From 6f274d476b95a8dff334292668b60c4058a0731f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=B0=E5=AE=9A=E4=B8=8D=E7=A6=BB=E6=89=8B?= <12640033+msz-006@user.noreply.gitee.com> Date: Mon, 14 Jul 2025 03:36:36 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=90=84=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0SQLite=E5=AD=98=E5=82=A8=E5=AE=9E=E7=8E=B0=E6=96=87?= =?UTF-8?q?=E4=BB=B6\n\n-=20=E6=96=B0=E5=A2=9E=20store/bilibili/bilibili?= =?UTF-8?q?=5Fstore=5Fimpl.py:=20B=E7=AB=99SQLite=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0\n-=20=E6=96=B0=E5=A2=9E=20store/douyin/douyi?= =?UTF-8?q?n=5Fstore=5Fimpl.py:=20=E6=8A=96=E9=9F=B3SQLite=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E5=AE=9E=E7=8E=B0\n-=20=E6=96=B0=E5=A2=9E=20store/kua?= =?UTF-8?q?ishou/kuaishou=5Fstore=5Fimpl.py:=20=E5=BF=AB=E6=89=8BSQLite?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=AE=9E=E7=8E=B0\n-=20=E6=96=B0=E5=A2=9E=20?= =?UTF-8?q?store/tieba/tieba=5Fstore=5Fimpl.py:=20=E8=B4=B4=E5=90=A7SQLite?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=AE=9E=E7=8E=B0\n-=20=E6=96=B0=E5=A2=9E=20?= =?UTF-8?q?store/weibo/weibo=5Fstore=5Fimpl.py:=20=E5=BE=AE=E5=8D=9ASQLite?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=AE=9E=E7=8E=B0\n-=20=E6=96=B0=E5=A2=9E=20?= =?UTF-8?q?store/xhs/xhs=5Fstore=5Fimpl.py:=20=E5=B0=8F=E7=BA=A2=E4=B9=A6S?= =?UTF-8?q?QLite=E5=AD=98=E5=82=A8=E5=AE=9E=E7=8E=B0\n-=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=20store/zhihu/zhihu=5Fstore=5Fimpl.py:=20=E7=9F=A5?= =?UTF-8?q?=E4=B9=8ESQLite=E5=AD=98=E5=82=A8=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- store/bilibili/bilibili_store_impl.py | 111 ++++++++++++++++++++++++++ store/douyin/douyin_store_impl.py | 66 ++++++++++++++- store/kuaishou/kuaishou_store_impl.py | 56 ++++++++++++- store/tieba/tieba_store_impl.py | 62 ++++++++++++++ store/weibo/weibo_store_impl.py | 64 +++++++++++++++ store/xhs/xhs_store_impl.py | 61 ++++++++++++++ store/zhihu/zhihu_store_impl.py | 62 ++++++++++++++ 7 files changed, 480 insertions(+), 2 deletions(-) diff --git a/store/bilibili/bilibili_store_impl.py b/store/bilibili/bilibili_store_impl.py index 0fa1504..d16a0a2 100644 --- a/store/bilibili/bilibili_store_impl.py +++ b/store/bilibili/bilibili_store_impl.py @@ -352,3 +352,114 @@ class BiliJsonStoreImplement(AbstractStore): """ await self.save_data_to_json(save_item=dynamic_item, store_type="dynamics") + + +class BiliSqliteStoreImplement(AbstractStore): + async def store_content(self, content_item: Dict): + """ + Bilibili content SQLite storage implementation + Args: + content_item: content item dict + + Returns: + + """ + + from .bilibili_store_sql import (add_new_content, + query_content_by_content_id, + update_content_by_content_id) + video_id = content_item.get("video_id") + video_detail: Dict = await query_content_by_content_id(content_id=video_id) + if not video_detail: + content_item["add_ts"] = utils.get_current_timestamp() + await add_new_content(content_item) + else: + await update_content_by_content_id(video_id, content_item=content_item) + + async def store_comment(self, comment_item: Dict): + """ + Bilibili comment SQLite storage implementation + Args: + comment_item: comment item dict + + Returns: + + """ + + from .bilibili_store_sql import (add_new_comment, + query_comment_by_comment_id, + update_comment_by_comment_id) + comment_id = comment_item.get("comment_id") + comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id) + if not comment_detail: + comment_item["add_ts"] = utils.get_current_timestamp() + await add_new_comment(comment_item) + else: + await update_comment_by_comment_id(comment_id, comment_item=comment_item) + + async def store_creator(self, creator: Dict): + """ + Bilibili creator SQLite storage implementation + Args: + creator: creator item dict + + Returns: + + """ + + from .bilibili_store_sql import (add_new_creator, + query_creator_by_creator_id, + update_creator_by_creator_id) + creator_id = creator.get("user_id") + creator_detail: Dict = await query_creator_by_creator_id(creator_id=creator_id) + if not creator_detail: + creator["add_ts"] = utils.get_current_timestamp() + await add_new_creator(creator) + else: + await update_creator_by_creator_id(creator_id, creator_item=creator) + + async def store_contact(self, contact_item: Dict): + """ + Bilibili contact SQLite storage implementation + Args: + contact_item: contact item dict + + Returns: + + """ + + from .bilibili_store_sql import (add_new_contact, + query_contact_by_up_and_fan, + update_contact_by_id, ) + + up_id = contact_item.get("up_id") + fan_id = contact_item.get("fan_id") + contact_detail: Dict = await query_contact_by_up_and_fan(up_id=up_id, fan_id=fan_id) + if not contact_detail: + contact_item["add_ts"] = utils.get_current_timestamp() + await add_new_contact(contact_item) + else: + key_id = contact_detail.get("id") + await update_contact_by_id(id=key_id, contact_item=contact_item) + + async def store_dynamic(self, dynamic_item): + """ + Bilibili dynamic SQLite storage implementation + Args: + dynamic_item: dynamic item dict + + Returns: + + """ + + from .bilibili_store_sql import (add_new_dynamic, + query_dynamic_by_dynamic_id, + update_dynamic_by_dynamic_id) + + dynamic_id = dynamic_item.get("dynamic_id") + dynamic_detail = await query_dynamic_by_dynamic_id(dynamic_id=dynamic_id) + if not dynamic_detail: + dynamic_item["add_ts"] = utils.get_current_timestamp() + await add_new_dynamic(dynamic_item) + else: + await update_dynamic_by_dynamic_id(dynamic_id, dynamic_item=dynamic_item) diff --git a/store/douyin/douyin_store_impl.py b/store/douyin/douyin_store_impl.py index 30c993c..8cd3196 100644 --- a/store/douyin/douyin_store_impl.py +++ b/store/douyin/douyin_store_impl.py @@ -257,4 +257,68 @@ class DouyinJsonStoreImplement(AbstractStore): Returns: """ - await self.save_data_to_json(save_item=creator, store_type="creator") \ No newline at end of file + await self.save_data_to_json(save_item=creator, store_type="creator") + + +class DouyinSqliteStoreImplement(AbstractStore): + async def store_content(self, content_item: Dict): + """ + Douyin content SQLite storage implementation + Args: + content_item: content item dict + + Returns: + + """ + + from .douyin_store_sql import (add_new_content, + query_content_by_content_id, + update_content_by_content_id) + aweme_id = content_item.get("aweme_id") + aweme_detail: Dict = await query_content_by_content_id(content_id=aweme_id) + if not aweme_detail: + content_item["add_ts"] = utils.get_current_timestamp() + if content_item.get("title"): + await add_new_content(content_item) + else: + await update_content_by_content_id(aweme_id, content_item=content_item) + + async def store_comment(self, comment_item: Dict): + """ + Douyin comment SQLite storage implementation + Args: + comment_item: comment item dict + + Returns: + + """ + from .douyin_store_sql import (add_new_comment, + query_comment_by_comment_id, + update_comment_by_comment_id) + comment_id = comment_item.get("comment_id") + comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id) + if not comment_detail: + comment_item["add_ts"] = utils.get_current_timestamp() + await add_new_comment(comment_item) + else: + await update_comment_by_comment_id(comment_id, comment_item=comment_item) + + async def store_creator(self, creator: Dict): + """ + Douyin creator SQLite storage implementation + Args: + creator: creator dict + + Returns: + + """ + from .douyin_store_sql import (add_new_creator, + query_creator_by_user_id, + update_creator_by_user_id) + user_id = creator.get("user_id") + user_detail: Dict = await query_creator_by_user_id(user_id) + if not user_detail: + creator["add_ts"] = utils.get_current_timestamp() + await add_new_creator(creator) + else: + await update_creator_by_user_id(user_id, creator) \ No newline at end of file diff --git a/store/kuaishou/kuaishou_store_impl.py b/store/kuaishou/kuaishou_store_impl.py index 4866796..950f3a2 100644 --- a/store/kuaishou/kuaishou_store_impl.py +++ b/store/kuaishou/kuaishou_store_impl.py @@ -233,4 +233,58 @@ class KuaishouJsonStoreImplement(AbstractStore): Returns: """ - await self.save_data_to_json(creator, "creator") \ No newline at end of file + await self.save_data_to_json(creator, "creator") + + +class KuaishouSqliteStoreImplement(AbstractStore): + async def store_content(self, content_item: Dict): + """ + Kuaishou content SQLite storage implementation + Args: + content_item: content item dict + + Returns: + + """ + + from .kuaishou_store_sql import (add_new_content, + query_content_by_content_id, + update_content_by_content_id) + video_id = content_item.get("video_id") + video_detail: Dict = await query_content_by_content_id(content_id=video_id) + if not video_detail: + content_item["add_ts"] = utils.get_current_timestamp() + await add_new_content(content_item) + else: + await update_content_by_content_id(video_id, content_item=content_item) + + async def store_comment(self, comment_item: Dict): + """ + Kuaishou comment SQLite storage implementation + Args: + comment_item: comment item dict + + Returns: + + """ + from .kuaishou_store_sql import (add_new_comment, + query_comment_by_comment_id, + update_comment_by_comment_id) + comment_id = comment_item.get("comment_id") + comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id) + if not comment_detail: + comment_item["add_ts"] = utils.get_current_timestamp() + await add_new_comment(comment_item) + else: + await update_comment_by_comment_id(comment_id, comment_item=comment_item) + + async def store_creator(self, creator: Dict): + """ + Kuaishou creator SQLite storage implementation + Args: + creator: creator dict + + Returns: + + """ + pass \ No newline at end of file diff --git a/store/tieba/tieba_store_impl.py b/store/tieba/tieba_store_impl.py index ff5da80..84267e4 100644 --- a/store/tieba/tieba_store_impl.py +++ b/store/tieba/tieba_store_impl.py @@ -254,3 +254,65 @@ class TieBaJsonStoreImplement(AbstractStore): """ await self.save_data_to_json(creator, "creator") + + +class TieBaSqliteStoreImplement(AbstractStore): + async def store_content(self, content_item: Dict): + """ + tieba content SQLite storage implementation + Args: + content_item: content item dict + + Returns: + + """ + from .tieba_store_sql import (add_new_content, + query_content_by_content_id, + update_content_by_content_id) + note_id = content_item.get("note_id") + note_detail: Dict = await query_content_by_content_id(content_id=note_id) + if not note_detail: + content_item["add_ts"] = utils.get_current_timestamp() + await add_new_content(content_item) + else: + await update_content_by_content_id(note_id, content_item=content_item) + + async def store_comment(self, comment_item: Dict): + """ + tieba comment SQLite storage implementation + Args: + comment_item: comment item dict + + Returns: + + """ + from .tieba_store_sql import (add_new_comment, + query_comment_by_comment_id, + update_comment_by_comment_id) + comment_id = comment_item.get("comment_id") + comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id) + if not comment_detail: + comment_item["add_ts"] = utils.get_current_timestamp() + await add_new_comment(comment_item) + else: + await update_comment_by_comment_id(comment_id, comment_item=comment_item) + + async def store_creator(self, creator: Dict): + """ + tieba creator SQLite storage implementation + Args: + creator: creator dict + + Returns: + + """ + from .tieba_store_sql import (add_new_creator, + query_creator_by_user_id, + update_creator_by_user_id) + user_id = creator.get("user_id") + user_detail: Dict = await query_creator_by_user_id(user_id) + if not user_detail: + creator["add_ts"] = utils.get_current_timestamp() + await add_new_creator(creator) + else: + await update_creator_by_user_id(user_id, creator) diff --git a/store/weibo/weibo_store_impl.py b/store/weibo/weibo_store_impl.py index 7348fec..2efd7f3 100644 --- a/store/weibo/weibo_store_impl.py +++ b/store/weibo/weibo_store_impl.py @@ -260,3 +260,67 @@ class WeiboJsonStoreImplement(AbstractStore): """ await self.save_data_to_json(creator, "creators") + + +class WeiboSqliteStoreImplement(AbstractStore): + async def store_content(self, content_item: Dict): + """ + Weibo content SQLite storage implementation + Args: + content_item: content item dict + + Returns: + + """ + + from .weibo_store_sql import (add_new_content, + query_content_by_content_id, + update_content_by_content_id) + note_id = content_item.get("note_id") + note_detail: Dict = await query_content_by_content_id(content_id=note_id) + if not note_detail: + content_item["add_ts"] = utils.get_current_timestamp() + await add_new_content(content_item) + else: + await update_content_by_content_id(note_id, content_item=content_item) + + async def store_comment(self, comment_item: Dict): + """ + Weibo comment SQLite storage implementation + Args: + comment_item: comment item dict + + Returns: + + """ + from .weibo_store_sql import (add_new_comment, + query_comment_by_comment_id, + update_comment_by_comment_id) + comment_id = comment_item.get("comment_id") + comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id) + if not comment_detail: + comment_item["add_ts"] = utils.get_current_timestamp() + await add_new_comment(comment_item) + else: + await update_comment_by_comment_id(comment_id, comment_item=comment_item) + + async def store_creator(self, creator: Dict): + """ + Weibo creator SQLite storage implementation + Args: + creator: + + Returns: + + """ + + from .weibo_store_sql import (add_new_creator, + query_creator_by_user_id, + update_creator_by_user_id) + user_id = creator.get("user_id") + user_detail: Dict = await query_creator_by_user_id(user_id) + if not user_detail: + creator["add_ts"] = utils.get_current_timestamp() + await add_new_creator(creator) + else: + await update_creator_by_user_id(user_id, creator) diff --git a/store/xhs/xhs_store_impl.py b/store/xhs/xhs_store_impl.py index 5ad4979..063b01b 100644 --- a/store/xhs/xhs_store_impl.py +++ b/store/xhs/xhs_store_impl.py @@ -255,3 +255,64 @@ class XhsJsonStoreImplement(AbstractStore): """ await self.save_data_to_json(creator, "creator") + + +class XhsSqliteStoreImplement(AbstractStore): + async def store_content(self, content_item: Dict): + """ + Xiaohongshu content SQLite storage implementation + Args: + content_item: content item dict + + Returns: + + """ + from .xhs_store_sql import (add_new_content, + query_content_by_content_id, + update_content_by_content_id) + note_id = content_item.get("note_id") + note_detail: Dict = await query_content_by_content_id(content_id=note_id) + if not note_detail: + content_item["add_ts"] = utils.get_current_timestamp() + await add_new_content(content_item) + else: + await update_content_by_content_id(note_id, content_item=content_item) + + async def store_comment(self, comment_item: Dict): + """ + Xiaohongshu comment SQLite storage implementation + Args: + comment_item: comment item dict + + Returns: + + """ + from .xhs_store_sql import (add_new_comment, + query_comment_by_comment_id, + update_comment_by_comment_id) + comment_id = comment_item.get("comment_id") + comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id) + if not comment_detail: + comment_item["add_ts"] = utils.get_current_timestamp() + await add_new_comment(comment_item) + else: + await update_comment_by_comment_id(comment_id, comment_item=comment_item) + + async def store_creator(self, creator: Dict): + """ + Xiaohongshu creator SQLite storage implementation + Args: + creator: creator dict + + Returns: + + """ + from .xhs_store_sql import (add_new_creator, query_creator_by_user_id, + update_creator_by_user_id) + user_id = creator.get("user_id") + user_detail: Dict = await query_creator_by_user_id(user_id) + if not user_detail: + creator["add_ts"] = utils.get_current_timestamp() + await add_new_creator(creator) + else: + await update_creator_by_user_id(user_id, creator) diff --git a/store/zhihu/zhihu_store_impl.py b/store/zhihu/zhihu_store_impl.py index 34d0a5b..84d5c06 100644 --- a/store/zhihu/zhihu_store_impl.py +++ b/store/zhihu/zhihu_store_impl.py @@ -254,3 +254,65 @@ class ZhihuJsonStoreImplement(AbstractStore): """ await self.save_data_to_json(creator, "creator") + + +class ZhihuSqliteStoreImplement(AbstractStore): + async def store_content(self, content_item: Dict): + """ + Zhihu content SQLite storage implementation + Args: + content_item: content item dict + + Returns: + + """ + from .zhihu_store_sql import (add_new_content, + query_content_by_content_id, + update_content_by_content_id) + note_id = content_item.get("note_id") + note_detail: Dict = await query_content_by_content_id(content_id=note_id) + if not note_detail: + content_item["add_ts"] = utils.get_current_timestamp() + await add_new_content(content_item) + else: + await update_content_by_content_id(note_id, content_item=content_item) + + async def store_comment(self, comment_item: Dict): + """ + Zhihu comment SQLite storage implementation + Args: + comment_item: comment item dict + + Returns: + + """ + from .zhihu_store_sql import (add_new_comment, + query_comment_by_comment_id, + update_comment_by_comment_id) + comment_id = comment_item.get("comment_id") + comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id) + if not comment_detail: + comment_item["add_ts"] = utils.get_current_timestamp() + await add_new_comment(comment_item) + else: + await update_comment_by_comment_id(comment_id, comment_item=comment_item) + + async def store_creator(self, creator: Dict): + """ + Zhihu creator SQLite storage implementation + Args: + creator: creator dict + + Returns: + + """ + from .zhihu_store_sql import (add_new_creator, + query_creator_by_user_id, + update_creator_by_user_id) + user_id = creator.get("user_id") + user_detail: Dict = await query_creator_by_user_id(user_id) + if not user_detail: + creator["add_ts"] = utils.get_current_timestamp() + await add_new_creator(creator) + else: + await update_creator_by_user_id(user_id, creator)