feat: weibo支持指定创作者主页

store/weibo/__init__.py

@@ -7,6 +7,7 @@ import re
 from typing import List
 
 from var import source_keyword_var
+
 from .weibo_store_image import *
 from .weibo_store_impl import *
 
@@ -27,7 +28,33 @@ class WeibostoreFactory:
         return store_class()
 
 
+async def batch_update_weibo_notes(note_list: List[Dict]):
+    """
+    Batch update weibo notes
+    Args:
+        note_list:
+
+    Returns:
+
+    """
+    if not note_list:
+        return
+    for note_item in note_list:
+        await update_weibo_note(note_item)
+
+
 async def update_weibo_note(note_item: Dict):
+    """
+    Update weibo note
+    Args:
+        note_item:
+
+    Returns:
+
+    """
+    if not note_item:
+        return
+
     mblog: Dict = note_item.get("mblog")
     user_info: Dict = mblog.get("user")
     note_id = mblog.get("id")
@@ -61,6 +88,15 @@ async def update_weibo_note(note_item: Dict):
 
 
 async def batch_update_weibo_note_comments(note_id: str, comments: List[Dict]):
+    """
+    Batch update weibo note comments
+    Args:
+        note_id:
+        comments:
+
+    Returns:
+
+    """
     if not comments:
         return
     for comment_item in comments:
@@ -68,6 +104,17 @@ async def batch_update_weibo_note_comments(note_id: str, comments: List[Dict]):
 
 
 async def update_weibo_note_comment(note_id: str, comment_item: Dict):
+    """
+    Update weibo note comment
+    Args:
+        note_id: weibo note id
+        comment_item: weibo comment item
+
+    Returns:
+
+    """
+    if not comment_item or not note_id:
+        return
     comment_id = str(comment_item.get("id"))
     user_info: Dict = comment_item.get("user")
     content_text = comment_item.get("text")
@@ -95,5 +142,43 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict):
         f"[store.weibo.update_weibo_note_comment] Weibo note comment: {comment_id}, content: {save_comment_item.get('content', '')[:24]} ...")
     await WeibostoreFactory.create_store().store_comment(comment_item=save_comment_item)
 
+
 async def update_weibo_note_image(picid: str, pic_content, extension_file_name):
-    await WeiboStoreImage().store_image({"pic_id": picid, "pic_content": pic_content, "extension_file_name": extension_file_name})
+    """
+    Save weibo note image to local
+    Args:
+        picid:
+        pic_content:
+        extension_file_name:
+
+    Returns:
+
+    """
+    await WeiboStoreImage().store_image(
+        {"pic_id": picid, "pic_content": pic_content, "extension_file_name": extension_file_name})
+
+
+async def save_creator(user_id: str, user_info: Dict):
+    """
+    Save creator information to local
+    Args:
+        user_id:
+        user_info:
+
+    Returns:
+
+    """
+    local_db_item = {
+        'user_id': user_id,
+        'nickname': user_info.get('screen_name'),
+        'gender': '女' if user_info.get('gender') == "f" else '男',
+        'avatar': user_info.get('avatar_hd'),
+        'desc': user_info.get('description'),
+        'ip_location': user_info.get("source", "").replace("来自", ""),
+        'follows': user_info.get('follow_count', ''),
+        'fans': user_info.get('followers_count', ''),
+        'tag_list': '',
+        "last_modify_ts": utils.get_current_timestamp(),
+    }
+    utils.logger.info(f"[store.weibo.save_creator] creator:{local_db_item}")
+    await WeibostoreFactory.create_store().store_creator(local_db_item)

store/weibo/weibo_store_impl.py

@@ -33,9 +33,6 @@ def calculate_number_of_files(file_store_path: str) -> int:
 
 
 class WeiboCsvStoreImplement(AbstractStore):
-    async def store_creator(self, creator: Dict):
-        pass
-
     csv_store_path: str = "data/weibo"
     file_count: int = calculate_number_of_files(csv_store_path)
 
@@ -91,6 +88,17 @@ class WeiboCsvStoreImplement(AbstractStore):
         """
         await self.save_data_to_csv(save_item=comment_item, store_type="comments")
 
+    async def store_creator(self, creator: Dict):
+        """
+        Weibo creator CSV storage implementation
+        Args:
+            creator:
+
+        Returns:
+
+        """
+        await self.save_data_to_csv(save_item=creator, store_type="creators")
+
 
 class WeiboDbStoreImplement(AbstractStore):
 
@@ -136,7 +144,25 @@ class WeiboDbStoreImplement(AbstractStore):
             await update_comment_by_comment_id(comment_id, comment_item=comment_item)
 
     async def store_creator(self, creator: Dict):
-        pass
+        """
+        Weibo creator DB storage implementation
+        Args:
+            creator:
+
+        Returns:
+
+        """
+
+        from .weibo_store_sql import (add_new_creator,
+                                      query_creator_by_user_id,
+                                      update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)
 
 
 class WeiboJsonStoreImplement(AbstractStore):
@@ -214,4 +240,12 @@ class WeiboJsonStoreImplement(AbstractStore):
         await self.save_data_to_json(comment_item, "comments")
 
     async def store_creator(self, creator: Dict):
-        pass
+        """
+        creator JSON storage implementation
+        Args:
+            creator:
+
+        Returns:
+
+        """
+        await self.save_data_to_json(creator, "creators")

store/weibo/weibo_store_sql.py

@@ -100,3 +100,49 @@ async def update_comment_by_comment_id(comment_id: str, comment_item: Dict) -> i
     async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
     effect_row: int = await async_db_conn.update_table("weibo_note_comment", comment_item, "comment_id", comment_id)
     return effect_row
+
+
+async def query_creator_by_user_id(user_id: str) -> Dict:
+    """
+    查询一条创作者记录
+    Args:
+        user_id:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    sql: str = f"select * from weibo_creator where user_id = '{user_id}'"
+    rows: List[Dict] = await async_db_conn.query(sql)
+    if len(rows) > 0:
+        return rows[0]
+    return dict()
+
+
+async def add_new_creator(creator_item: Dict) -> int:
+    """
+    新增一条创作者信息
+    Args:
+        creator_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    last_row_id: int = await async_db_conn.item_to_table("weibo_creator", creator_item)
+    return last_row_id
+
+
+async def update_creator_by_user_id(user_id: str, creator_item: Dict) -> int:
+    """
+    更新一条创作者信息
+    Args:
+        user_id:
+        creator_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    effect_row: int = await async_db_conn.update_table("weibo_creator", creator_item, "user_id", user_id)
+    return effect_row