feat: 支持爬取小红书二级评论

store/xhs/__init__.py

@@ -89,6 +89,34 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
     }
     utils.logger.info(f"[store.xhs.update_xhs_note_comment] xhs note comment:{local_db_item}")
     await XhsStoreFactory.create_store().store_comment(local_db_item)
+    
+async def batch_update_xhs_note_sub_comments(note_id: str, comments: List[Dict]):
+    if not comments:
+        return
+    for comment_item in comments:
+        await update_xhs_note_sub_comment(note_id, comment_item)
+
+
+async def update_xhs_note_sub_comment(note_id: str, comment_item: Dict):
+    user_info = comment_item.get("user_info", {})
+    target_comment = comment_item.get("target_comment")
+    comment_id = comment_item.get("id")
+    comment_pictures = [item.get("url_default", "") for item in comment_item.get("pictures", [])]
+    local_db_item = {
+        "comment_id": comment_id,
+        "target_comment_id": target_comment.get('id'),
+        "create_time": comment_item.get("create_time"),
+        "ip_location": comment_item.get("ip_location"),
+        "note_id": note_id,
+        "content": comment_item.get("content"),
+        "user_id": user_info.get("user_id"),
+        "nickname": user_info.get("nickname"),
+        "avatar": user_info.get("image"),
+        "pictures": ",".join(comment_pictures),
+        "last_modify_ts": utils.get_current_timestamp(),
+    }
+    utils.logger.info(f"[store.xhs.update_xhs_note_sub_comment] xhs note comment:{local_db_item}")
+    await XhsStoreFactory.create_store().store_sub_comment(local_db_item)
 
 
 async def save_creator(user_id: str, creator: Dict):

store/xhs/xhs_store_impl.py

@@ -81,6 +81,17 @@ class XhsCsvStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_csv(save_item=creator, store_type="creator")
+    
+    async def store_sub_comment(self, sub_comment_items: Dict):
+        """
+        Xiaohongshu sub_comments CSV storage implementation
+        Args:
+            sub_comment_items: sub_comments item dict
+
+        Returns:
+
+        """
+        await self.save_data_to_csv(save_item=sub_comment_items, store_type="sub_comments")
 
 
 class XhsDbStoreImplement(AbstractStore):
@@ -142,6 +153,26 @@ class XhsDbStoreImplement(AbstractStore):
             await add_new_creator(creator)
         else:
             await update_creator_by_user_id(user_id, creator)
+            
+    async def store_sub_comment(self, sub_comment_items: Dict):
+        """
+        Xiaohongshu content DB storage implementation
+        Args:
+            sub_comment_items: comment item dict
+
+        Returns:
+
+        """
+        from .xhs_store_sql import (add_new_sub_comment,
+                                    query_sub_comment_by_comment_id,
+                                    update_sub_comment_by_comment_id)
+        comment_id = sub_comment_items.get("comment_id")
+        comment_detail: Dict = await query_sub_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            sub_comment_items["add_ts"] = utils.get_current_timestamp()
+            await add_new_sub_comment(sub_comment_items)
+        else:
+            await update_sub_comment_by_comment_id(comment_id, comment_item=sub_comment_items)
 
 
 class XhsJsonStoreImplement(AbstractStore):
@@ -214,3 +245,14 @@ class XhsJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(creator, "creator")
+        
+    async def store_sub_comment(self, sub_comment_items: Dict):
+        """
+        sub_comment JSON storage implementatio
+        Args:
+            sub_comment_items:
+
+        Returns:
+
+        """
+        await self.save_data_to_json(sub_comment_items, "sub_comments")

store/xhs/xhs_store_sql.py

@@ -145,4 +145,50 @@ async def update_creator_by_user_id(user_id: str, creator_item: Dict) -> int:
     """
     async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
     effect_row: int = await async_db_conn.update_table("xhs_creator", creator_item, "user_id", user_id)
+    return effect_row
+
+
+async def query_sub_comment_by_comment_id(comment_id: str) -> Dict:
+    """
+    查询一条二级评论内容
+    Args:
+        comment_id:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    sql: str = f"select * from xhs_note_sub_comment where comment_id = '{comment_id}'"
+    rows: List[Dict] = await async_db_conn.query(sql)
+    if len(rows) > 0:
+        return rows[0]
+    return dict()
+
+
+async def add_new_sub_comment(comment_item: Dict) -> int:
+    """
+    新增一条二级评论记录
+    Args:
+        comment_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    last_row_id: int = await async_db_conn.item_to_table("xhs_note_sub_comment", comment_item)
+    return last_row_id
+
+
+async def update_sub_comment_by_comment_id(comment_id: str, comment_item: Dict) -> int:
+    """
+    更新增一条二级评论记录
+    Args:
+        comment_id:
+        comment_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    effect_row: int = await async_db_conn.update_table("xhs_note_sub_comment", comment_item, "comment_id", comment_id)
     return effect_row