jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 19:37:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: weibo支持指定创作者主页

Browse Source
This commit is contained in:
Relakkes
2024-08-24 05:52:11 +08:00
parent 61f023edac
commit ab7d8142af
9 changed files with 368 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
store/weibo/weibo_store_impl.py
View File

@@ -33,9 +33,6 @@ def calculate_number_of_files(file_store_path: str) -> int:
class WeiboCsvStoreImplement(AbstractStore):
async def store_creator(self, creator: Dict):
pass
csv_store_path: str = "data/weibo"
file_count: int = calculate_number_of_files(csv_store_path)
@@ -91,6 +88,17 @@ class WeiboCsvStoreImplement(AbstractStore):
"""
await self.save_data_to_csv(save_item=comment_item, store_type="comments")
async def store_creator(self, creator: Dict):
"""
Weibo creator CSV storage implementation
Args:
creator:
Returns:
"""
await self.save_data_to_csv(save_item=creator, store_type="creators")
class WeiboDbStoreImplement(AbstractStore):
@@ -136,7 +144,25 @@ class WeiboDbStoreImplement(AbstractStore):
await update_comment_by_comment_id(comment_id, comment_item=comment_item)
async def store_creator(self, creator: Dict):
pass
"""
Weibo creator DB storage implementation
Args:
creator:
Returns:
"""
from .weibo_store_sql import (add_new_creator,
query_creator_by_user_id,
update_creator_by_user_id)
user_id = creator.get("user_id")
user_detail: Dict = await query_creator_by_user_id(user_id)
if not user_detail:
creator["add_ts"] = utils.get_current_timestamp()
await add_new_creator(creator)
else:
await update_creator_by_user_id(user_id, creator)
class WeiboJsonStoreImplement(AbstractStore):
@@ -214,4 +240,12 @@ class WeiboJsonStoreImplement(AbstractStore):
await self.save_data_to_json(comment_item, "comments")
async def store_creator(self, creator: Dict):
pass
"""
creator JSON storage implementation
Args:
creator:
Returns:
"""
await self.save_data_to_json(creator, "creators")