feat: 抖音指定创作者done

store/douyin/__init__.py

@@ -92,3 +92,31 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
         f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}")
 
     await DouyinStoreFactory.create_store().store_comment(comment_item=save_comment_item)
+
+
+
+
+async def save_creator(user_id: str, creator: Dict):
+    user_info = creator.get('user', {})
+    gender_map = {
+        0: '未知',
+        1: '男',
+        2: '女'
+    }
+    avatar_uri = user_info.get('avatar_300x300', {}).get('uri')
+    local_db_item = {
+        'user_id': user_id,
+        'nickname': user_info.get('nickname'),
+        'gender': gender_map.get(user_info.get('gender'), '未知'),
+        'avatar': f"https://p3-pc.douyinpic.com/img/{avatar_uri}" + r"~c5_300x300.jpeg?from=2956013662",
+        'desc': user_info.get('signature'),
+        'ip_location': user_info.get('ip_location'),
+        'follows': user_info.get("following_count", 0),
+        'fans': user_info.get("max_follower_count", 0),
+        'interaction': user_info.get("total_favorited", 0),
+        'videos_count': user_info.get("aweme_count", 0),
+        "last_modify_ts": utils.get_current_timestamp(),
+
+    }
+    utils.logger.info(f"[store.douyin.save_creator] creator:{local_db_item}")
+    await DouyinStoreFactory.create_store().store_creator(local_db_item)

store/douyin/douyin_store_impl.py

@@ -26,13 +26,14 @@ def calculate_number_of_files(file_store_path: str) -> int:
     if not os.path.exists(file_store_path):
         return 1
     try:
-        return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
+        return max([int(file_name.split("_")[0]) for file_name in os.listdir(file_store_path)]) + 1
     except ValueError:
         return 1
 
+
 class DouyinCsvStoreImplement(AbstractStore):
     csv_store_path: str = "data/douyin"
-    file_count:int=calculate_number_of_files(csv_store_path)
+    file_count: int = calculate_number_of_files(csv_store_path)
 
     def make_save_file_name(self, store_type: str) -> str:
         """
@@ -65,7 +66,7 @@ class DouyinCsvStoreImplement(AbstractStore):
 
     async def store_content(self, content_item: Dict):
         """
-        Xiaohongshu content CSV storage implementation
+        Douyin content CSV storage implementation
         Args:
             content_item: note item dict
 
@@ -76,7 +77,7 @@ class DouyinCsvStoreImplement(AbstractStore):
 
     async def store_comment(self, comment_item: Dict):
         """
-        Xiaohongshu comment CSV storage implementation
+        Douyin comment CSV storage implementation
         Args:
             comment_item: comment item dict
 
@@ -85,6 +86,17 @@ class DouyinCsvStoreImplement(AbstractStore):
         """
         await self.save_data_to_csv(save_item=comment_item, store_type="comments")
 
+    async def store_creator(self, creator: Dict):
+        """
+        Douyin creator CSV storage implementation
+        Args:
+            creator: creator item dict
+
+        Returns:
+
+        """
+        await self.save_data_to_csv(save_item=creator, store_type="creator")
+
 
 class DouyinDbStoreImplement(AbstractStore):
     async def store_content(self, content_item: Dict):
@@ -109,7 +121,6 @@ class DouyinDbStoreImplement(AbstractStore):
         else:
             await update_content_by_content_id(aweme_id, content_item=content_item)
 
-
     async def store_comment(self, comment_item: Dict):
         """
         Douyin content DB storage implementation
@@ -130,11 +141,29 @@ class DouyinDbStoreImplement(AbstractStore):
         else:
             await update_comment_by_comment_id(comment_id, comment_item=comment_item)
 
+    async def store_creator(self, creator: Dict):
+        """
+        Douyin content DB storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        from .douyin_store_sql import (add_new_creator, query_creator_by_user_id,
+                                    update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)
 
 class DouyinJsonStoreImplement(AbstractStore):
     json_store_path: str = "data/douyin"
     lock = asyncio.Lock()
-    file_count:int=calculate_number_of_files(json_store_path)
+    file_count: int = calculate_number_of_files(json_store_path)
 
     def make_save_file_name(self, store_type: str) -> str:
         """
@@ -146,7 +175,6 @@ class DouyinJsonStoreImplement(AbstractStore):
 
         """
 
-
         return f"{self.json_store_path}/{self.file_count}_{crawler_type_var.get()}_{store_type}_{utils.get_current_date()}.json"
 
     async def save_data_to_json(self, save_item: Dict, store_type: str):
@@ -193,3 +221,15 @@ class DouyinJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(comment_item, "comments")
+
+
+    async def store_creator(self, creator: Dict):
+        """
+        Douyin creator CSV storage implementation
+        Args:
+            creator: creator item dict
+
+        Returns:
+
+        """
+        await self.save_data_to_json(save_item=creator, store_type="creator")

store/douyin/douyin_store_sql.py

@@ -100,3 +100,49 @@ async def update_comment_by_comment_id(comment_id: str, comment_item: Dict) -> i
     async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
     effect_row: int = await async_db_conn.update_table("douyin_aweme_comment", comment_item, "comment_id", comment_id)
     return effect_row
+
+
+async def query_creator_by_user_id(user_id: str) -> Dict:
+    """
+    查询一条创作者记录
+    Args:
+        user_id:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    sql: str = f"select * from dy_creator where user_id = '{user_id}'"
+    rows: List[Dict] = await async_db_conn.query(sql)
+    if len(rows) > 0:
+        return rows[0]
+    return dict()
+
+
+async def add_new_creator(creator_item: Dict) -> int:
+    """
+    新增一条创作者信息
+    Args:
+        creator_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    last_row_id: int = await async_db_conn.item_to_table("dy_creator", creator_item)
+    return last_row_id
+
+
+async def update_creator_by_user_id(user_id: str, creator_item: Dict) -> int:
+    """
+    更新一条创作者信息
+    Args:
+        user_id:
+        creator_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    effect_row: int = await async_db_conn.update_table("dy_creator", creator_item, "user_id", user_id)
+    return effect_row