feat: 小红书支持通过博主ID采集笔记和评论，小红书type=search时支持配置按哪种排序方式获取笔记数据，小红书笔记增加视频地址和标签字段

store/xhs/__init__.py

@@ -31,12 +31,20 @@ async def update_xhs_note(note_item: Dict):
     user_info = note_item.get("user", {})
     interact_info = note_item.get("interact_info", {})
     image_list: List[Dict] = note_item.get("image_list", [])
+    tag_list: List[Dict] = note_item.get("tag_list", [])
+
+    video_url = ''
+    if note_item.get('type') == 'video':
+        videos = note_item.get('video').get('media').get('stream').get('h264')
+        if type(videos).__name__ == 'list':
+            video_url = ','.join([ v.get('master_url') for v in videos])
 
     local_db_item = {
         "note_id": note_item.get("note_id"),
         "type": note_item.get("type"),
         "title": note_item.get("title") or note_item.get("desc", "")[:255],
         "desc": note_item.get("desc", ""),
+        "video_url": video_url,
         "time": note_item.get("time"),
         "last_update_time": note_item.get("last_update_time", 0),
         "user_id": user_info.get("user_id"),
@@ -48,6 +56,7 @@ async def update_xhs_note(note_item: Dict):
         "share_count": interact_info.get("share_count"),
         "ip_location": note_item.get("ip_location", ""),
         "image_list": ','.join([img.get('url', '') for img in image_list]),
+        "tag_list": ','.join([tag.get('name', '') for tag in tag_list if tag.get('type')=='topic']),
         "last_modify_ts": utils.get_current_timestamp(),
         "note_url": f"https://www.xiaohongshu.com/explore/{note_id}"
     }
@@ -77,3 +86,32 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
     }
     utils.logger.info(f"[store.xhs.update_xhs_note_comment] xhs note comment:{local_db_item}")
     await XhsStoreFactory.create_store().store_comment(local_db_item)
+
+async def save_creator(user_id: str, creator: Dict):
+    user_info = creator.get('basicInfo', {})
+
+    follows = 0
+    fans = 0
+    interaction = 0
+    for i in creator.get('interactions'):
+        if i.get('type') == 'follows':
+            follows = i.get('count')
+        elif i.get('type') == 'fans':
+            fans = i.get('count')
+        elif i.get('type') == 'interaction':
+            interaction = i.get('count')
+
+    local_db_item = {
+        'user_id': user_id,
+        'nickname': user_info.get('nickname'),
+        'gender': '女' if user_info.get('gender') == 1 else '男' ,
+        'avatar': user_info.get('images'),
+        'desc': user_info.get('desc'),
+        'ip_location': user_info.get('ip_location'),
+        'follows': follows,
+        'fans': fans,
+        'interaction': interaction,
+        'tag_list': json.dumps({tag.get('tagType'):tag.get('name') for tag in creator.get('tags')}),
+    }
+    utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}")
+    await XhsStoreFactory.create_store().store_creator(local_db_item)

store/xhs/xhs_store_db_types.py

@@ -25,6 +25,7 @@ class XHSNote(XhsBaseModel):
     type = fields.CharField(null=True, max_length=16, description="笔记类型(normal | video)")
     title = fields.CharField(null=True, max_length=255, description="笔记标题")
     desc = fields.TextField(null=True, description="笔记描述")
+    video_url = fields.TextField(null=True, description="视频地址")
     time = fields.BigIntField(description="笔记发布时间戳", index=True)
     last_update_time = fields.BigIntField(description="笔记最后更新时间戳")
     liked_count = fields.CharField(null=True, max_length=16, description="笔记点赞数")
@@ -32,6 +33,7 @@ class XHSNote(XhsBaseModel):
     comment_count = fields.CharField(null=True, max_length=16, description="笔记评论数")
     share_count = fields.CharField(null=True, max_length=16, description="笔记分享数")
     image_list = fields.TextField(null=True, description="笔记封面图片列表")
+    tag_list = fields.TextField(null=True, description="标签列表")
     note_url = fields.CharField(null=True, max_length=255, description="笔记详情页的URL")
 
     class Meta:
@@ -55,3 +57,19 @@ class XHSNoteComment(XhsBaseModel):
 
     def __str__(self):
         return f"{self.comment_id} - {self.content}"
+
+
+class XhsCreator(XhsBaseModel):
+    desc = fields.TextField(null=True, description="用户描述")
+    gender = fields.CharField(null=True, max_length=1, description="性别")
+    follows = fields.CharField(null=True, max_length=16, description="关注数")
+    fans = fields.CharField(null=True, max_length=16, description="粉丝数")
+    interaction = fields.CharField(null=True, max_length=16, description="获赞和收藏数")
+    # follows = fields.IntField(description="关注数")
+    # fans = fields.IntField(description="粉丝数")
+    # interaction = fields.IntField(description="获赞和收藏数")
+    tag_list = fields.TextField(null=True, description="标签列表") # json字符串
+
+    class Meta:
+        table = "xhs_creator"
+        table_description = "小红书博主"

store/xhs/xhs_store_impl.py

@@ -72,6 +72,17 @@ class XhsCsvStoreImplement(AbstractStore):
         """
         await self.save_data_to_csv(save_item=comment_item, store_type="comments")
 
+    async def store_creator(self, creator: Dict):
+        """
+        Xiaohongshu content CSV storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        await self.save_data_to_csv(save_item=creator, store_type="creator")
+
 
 class XhsDbStoreImplement(AbstractStore):
     async def store_content(self, content_item: Dict):
@@ -121,6 +132,31 @@ class XhsDbStoreImplement(AbstractStore):
             comment_pydantic.model_validate(comment_data)
             await XHSNoteComment.filter(comment_id=comment_id).update(**comment_data.model_dump())
 
+    async def store_creator(self, creator: Dict):
+        """
+        Xiaohongshu content DB storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        from .xhs_store_db_types import XhsCreator
+        user_id = creator.get("user_id")
+        if not await XhsCreator.filter(user_id=user_id).first():
+            creator["add_ts"] = utils.get_current_timestamp()
+            creator["last_modify_ts"] = creator["add_ts"]
+            creator_pydantic = pydantic_model_creator(XhsCreator, name="CreatorPydanticCreate", exclude=('id',))
+            creator_data = creator_pydantic(**creator)
+            creator_pydantic.model_validate(creator_data)
+            await XhsCreator.create(**creator_data.model_dump())
+        else:
+            creator["last_modify_ts"] = utils.get_current_timestamp()
+            creator_pydantic = pydantic_model_creator(XhsCreator, name="CreatorPydanticUpdate", exclude=('id', 'add_ts',))
+            creator_data = creator_pydantic(**creator)
+            creator_pydantic.model_validate(creator_data)
+            await XhsCreator.filter(user_id=user_id).update(**creator_data.model_dump())
+
 
 class XhsJsonStoreImplement(AbstractStore):
     json_store_path: str = "data/xhs"
@@ -181,3 +217,14 @@ class XhsJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(comment_item, "comments")
+    
+    async def store_creator(self, creator: Dict):
+        """
+        Xiaohongshu content JSON storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        await self.save_data_to_json(creator, "creator")