feat: db数据存储重构完成

store/weibo/weibo_store_impl.py

@@ -82,20 +82,17 @@ class WeiboDbStoreImplement(AbstractStore):
         Returns:
 
         """
-        from .weibo_store_db_types import WeiboNote
+
+        from .weibo_store_sql import (add_new_content,
+                                      query_content_by_content_id,
+                                      update_content_by_content_id)
         note_id = content_item.get("note_id")
-        if not await WeiboNote.filter(note_id=note_id).exists():
+        note_detail: Dict = await query_content_by_content_id(content_id=note_id)
+        if not note_detail:
             content_item["add_ts"] = utils.get_current_timestamp()
-            weibo_note_pydantic = pydantic_model_creator(WeiboNote, name='WeiboNoteCreate', exclude=('id',))
-            weibo_data = weibo_note_pydantic(**content_item)
-            weibo_note_pydantic.model_validate(weibo_data)
-            await WeiboNote.create(**weibo_data.model_dump())
+            await add_new_content(content_item)
         else:
-            weibo_note_pydantic = pydantic_model_creator(WeiboNote, name='WeiboNoteUpdate',
-                                                         exclude=('id', 'add_ts'))
-            weibo_data = weibo_note_pydantic(**content_item)
-            weibo_note_pydantic.model_validate(weibo_data)
-            await WeiboNote.filter(note_id=note_id).update(**weibo_data.model_dump())
+            await update_content_by_content_id(note_id, content_item=content_item)
 
     async def store_comment(self, comment_item: Dict):
         """
@@ -106,21 +103,16 @@ class WeiboDbStoreImplement(AbstractStore):
         Returns:
 
         """
-        from .weibo_store_db_types import WeiboComment
+        from .weibo_store_sql import (add_new_comment,
+                                      query_comment_by_comment_id,
+                                      update_comment_by_comment_id)
         comment_id = comment_item.get("comment_id")
-        if not await WeiboComment.filter(comment_id=comment_id).exists():
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
             comment_item["add_ts"] = utils.get_current_timestamp()
-            comment_pydantic = pydantic_model_creator(WeiboComment, name='WeiboNoteCommentCreate',
-                                                      exclude=('id',))
-            comment_data = comment_pydantic(**comment_item)
-            comment_pydantic.model_validate(comment_data)
-            await WeiboComment.create(**comment_data.model_dump())
+            await add_new_comment(comment_item)
         else:
-            comment_pydantic = pydantic_model_creator(WeiboComment, name='WeiboNoteCommentUpdate',
-                                                      exclude=('id', 'add_ts'))
-            comment_data = comment_pydantic(**comment_item)
-            comment_pydantic.model_validate(comment_data)
-            await WeiboComment.filter(comment_id=comment_id).update(**comment_data.model_dump())
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
 
 
 class WeiboJsonStoreImplement(AbstractStore):
@@ -161,7 +153,6 @@ class WeiboJsonStoreImplement(AbstractStore):
             async with aiofiles.open(save_file_name, 'w', encoding='utf-8') as file:
                 await file.write(json.dumps(save_data, ensure_ascii=False))
 
-
     async def store_content(self, content_item: Dict):
         """
         content JSON storage implementation

store/weibo/weibo_store_sql.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2024/4/6 15:30
+# @Desc    : sql接口集合
+
+from typing import Dict, List
+
+from db import AsyncMysqlDB
+from var import media_crawler_db_var
+
+
+async def query_content_by_content_id(content_id: str) -> Dict:
+    """
+    查询一条内容记录（xhs的帖子 ｜ 抖音的视频 ｜ 微博 ｜ 快手视频 ...）
+    Args:
+        content_id:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    sql: str = f"select * from weibo_note where note_id = '{content_id}'"
+    rows: List[Dict] = await async_db_conn.query(sql)
+    if len(rows) > 0:
+        return rows[0]
+    return dict()
+
+
+async def add_new_content(content_item: Dict) -> int:
+    """
+    新增一条内容记录（xhs的帖子 ｜ 抖音的视频 ｜ 微博 ｜ 快手视频 ...）
+    Args:
+        content_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    last_row_id: int = await async_db_conn.item_to_table("weibo_note", content_item)
+    return last_row_id
+
+
+async def update_content_by_content_id(content_id: str, content_item: Dict) -> int:
+    """
+    更新一条记录（xhs的帖子 ｜ 抖音的视频 ｜ 微博 ｜ 快手视频 ...）
+    Args:
+        content_id:
+        content_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    effect_row: int = await async_db_conn.update_table("weibo_note", content_item, "note_id", content_id)
+    return effect_row
+
+
+
+async def query_comment_by_comment_id(comment_id: str) -> Dict:
+    """
+    查询一条评论内容
+    Args:
+        comment_id:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    sql: str = f"select * from weibo_note_comment where comment_id = '{comment_id}'"
+    rows: List[Dict] = await async_db_conn.query(sql)
+    if len(rows) > 0:
+        return rows[0]
+    return dict()
+
+
+async def add_new_comment(comment_item: Dict) -> int:
+    """
+    新增一条评论记录
+    Args:
+        comment_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    last_row_id: int = await async_db_conn.item_to_table("weibo_note_comment", comment_item)
+    return last_row_id
+
+
+async def update_comment_by_comment_id(comment_id: str, comment_item: Dict) -> int:
+    """
+    更新增一条评论记录
+    Args:
+        comment_id:
+        comment_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    effect_row: int = await async_db_conn.update_table("weibo_note_comment", comment_item, "comment_id", comment_id)
+    return effect_row