feat: 添加各平台SQLite存储实现文件\n\n- 新增 store/bilibili/bilibili_store_impl.py: B站SQLite存储实现\n- 新增 store/douyin/douyin_store_impl.py: 抖音SQLite存储实现\n- 新增 store/kuaishou/kuaishou_store_impl.py: 快手SQLite存储实现\n- 新增 store/tieba/tieba_store_impl.py: 贴吧SQLite存储实现\n- 新增 store/weibo/weibo_store_impl.py: 微博SQLite存储实现\n- 新增 store/xhs/xhs_store_impl.py: 小红书SQLite存储实现\n- 新增 store/zhihu/zhihu_store_impl.py: 知乎SQLite存储实现

store/bilibili/bilibili_store_impl.py

@@ -352,3 +352,114 @@ class BiliJsonStoreImplement(AbstractStore):
         """
 
         await self.save_data_to_json(save_item=dynamic_item, store_type="dynamics")
+
+
+class BiliSqliteStoreImplement(AbstractStore):
+    async def store_content(self, content_item: Dict):
+        """
+        Bilibili content SQLite storage implementation
+        Args:
+            content_item: content item dict
+
+        Returns:
+
+        """
+
+        from .bilibili_store_sql import (add_new_content,
+                                         query_content_by_content_id,
+                                         update_content_by_content_id)
+        video_id = content_item.get("video_id")
+        video_detail: Dict = await query_content_by_content_id(content_id=video_id)
+        if not video_detail:
+            content_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_content(content_item)
+        else:
+            await update_content_by_content_id(video_id, content_item=content_item)
+
+    async def store_comment(self, comment_item: Dict):
+        """
+        Bilibili comment SQLite storage implementation
+        Args:
+            comment_item: comment item dict
+
+        Returns:
+
+        """
+
+        from .bilibili_store_sql import (add_new_comment,
+                                         query_comment_by_comment_id,
+                                         update_comment_by_comment_id)
+        comment_id = comment_item.get("comment_id")
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            comment_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_comment(comment_item)
+        else:
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
+
+    async def store_creator(self, creator: Dict):
+        """
+        Bilibili creator SQLite storage implementation
+        Args:
+            creator: creator item dict
+
+        Returns:
+
+        """
+
+        from .bilibili_store_sql import (add_new_creator,
+                                         query_creator_by_creator_id,
+                                         update_creator_by_creator_id)
+        creator_id = creator.get("user_id")
+        creator_detail: Dict = await query_creator_by_creator_id(creator_id=creator_id)
+        if not creator_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_creator_id(creator_id, creator_item=creator)
+
+    async def store_contact(self, contact_item: Dict):
+        """
+        Bilibili contact SQLite storage implementation
+        Args:
+            contact_item: contact item dict
+
+        Returns:
+
+        """
+
+        from .bilibili_store_sql import (add_new_contact,
+                                         query_contact_by_up_and_fan,
+                                         update_contact_by_id, )
+
+        up_id = contact_item.get("up_id")
+        fan_id = contact_item.get("fan_id")
+        contact_detail: Dict = await query_contact_by_up_and_fan(up_id=up_id, fan_id=fan_id)
+        if not contact_detail:
+            contact_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_contact(contact_item)
+        else:
+            key_id = contact_detail.get("id")
+            await update_contact_by_id(id=key_id, contact_item=contact_item)
+
+    async def store_dynamic(self, dynamic_item):
+        """
+        Bilibili dynamic SQLite storage implementation
+        Args:
+            dynamic_item: dynamic item dict
+
+        Returns:
+
+        """
+
+        from .bilibili_store_sql import (add_new_dynamic,
+                                         query_dynamic_by_dynamic_id,
+                                         update_dynamic_by_dynamic_id)
+
+        dynamic_id = dynamic_item.get("dynamic_id")
+        dynamic_detail = await query_dynamic_by_dynamic_id(dynamic_id=dynamic_id)
+        if not dynamic_detail:
+            dynamic_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_dynamic(dynamic_item)
+        else:
+            await update_dynamic_by_dynamic_id(dynamic_id, dynamic_item=dynamic_item)

store/douyin/douyin_store_impl.py

@@ -257,4 +257,68 @@ class DouyinJsonStoreImplement(AbstractStore):
         Returns:
 
         """
-        await self.save_data_to_json(save_item=creator, store_type="creator")
+        await self.save_data_to_json(save_item=creator, store_type="creator")
+
+
+class DouyinSqliteStoreImplement(AbstractStore):
+    async def store_content(self, content_item: Dict):
+        """
+        Douyin content SQLite storage implementation
+        Args:
+            content_item: content item dict
+
+        Returns:
+
+        """
+
+        from .douyin_store_sql import (add_new_content,
+                                       query_content_by_content_id,
+                                       update_content_by_content_id)
+        aweme_id = content_item.get("aweme_id")
+        aweme_detail: Dict = await query_content_by_content_id(content_id=aweme_id)
+        if not aweme_detail:
+            content_item["add_ts"] = utils.get_current_timestamp()
+            if content_item.get("title"):
+                await add_new_content(content_item)
+        else:
+            await update_content_by_content_id(aweme_id, content_item=content_item)
+
+    async def store_comment(self, comment_item: Dict):
+        """
+        Douyin comment SQLite storage implementation
+        Args:
+            comment_item: comment item dict
+
+        Returns:
+
+        """
+        from .douyin_store_sql import (add_new_comment,
+                                       query_comment_by_comment_id,
+                                       update_comment_by_comment_id)
+        comment_id = comment_item.get("comment_id")
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            comment_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_comment(comment_item)
+        else:
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
+
+    async def store_creator(self, creator: Dict):
+        """
+        Douyin creator SQLite storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        from .douyin_store_sql import (add_new_creator,
+                                       query_creator_by_user_id,
+                                       update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)

store/kuaishou/kuaishou_store_impl.py

@@ -233,4 +233,58 @@ class KuaishouJsonStoreImplement(AbstractStore):
         Returns:
 
         """
-        await self.save_data_to_json(creator, "creator")
+        await self.save_data_to_json(creator, "creator")
+
+
+class KuaishouSqliteStoreImplement(AbstractStore):
+    async def store_content(self, content_item: Dict):
+        """
+        Kuaishou content SQLite storage implementation
+        Args:
+            content_item: content item dict
+
+        Returns:
+
+        """
+
+        from .kuaishou_store_sql import (add_new_content,
+                                         query_content_by_content_id,
+                                         update_content_by_content_id)
+        video_id = content_item.get("video_id")
+        video_detail: Dict = await query_content_by_content_id(content_id=video_id)
+        if not video_detail:
+            content_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_content(content_item)
+        else:
+            await update_content_by_content_id(video_id, content_item=content_item)
+
+    async def store_comment(self, comment_item: Dict):
+        """
+        Kuaishou comment SQLite storage implementation
+        Args:
+            comment_item: comment item dict
+
+        Returns:
+
+        """
+        from .kuaishou_store_sql import (add_new_comment,
+                                         query_comment_by_comment_id,
+                                         update_comment_by_comment_id)
+        comment_id = comment_item.get("comment_id")
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            comment_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_comment(comment_item)
+        else:
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
+
+    async def store_creator(self, creator: Dict):
+        """
+        Kuaishou creator SQLite storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        pass

store/tieba/tieba_store_impl.py

@@ -254,3 +254,65 @@ class TieBaJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(creator, "creator")
+
+
+class TieBaSqliteStoreImplement(AbstractStore):
+    async def store_content(self, content_item: Dict):
+        """
+        tieba content SQLite storage implementation
+        Args:
+            content_item: content item dict
+
+        Returns:
+
+        """
+        from .tieba_store_sql import (add_new_content,
+                                      query_content_by_content_id,
+                                      update_content_by_content_id)
+        note_id = content_item.get("note_id")
+        note_detail: Dict = await query_content_by_content_id(content_id=note_id)
+        if not note_detail:
+            content_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_content(content_item)
+        else:
+            await update_content_by_content_id(note_id, content_item=content_item)
+
+    async def store_comment(self, comment_item: Dict):
+        """
+        tieba comment SQLite storage implementation
+        Args:
+            comment_item: comment item dict
+
+        Returns:
+
+        """
+        from .tieba_store_sql import (add_new_comment,
+                                      query_comment_by_comment_id,
+                                      update_comment_by_comment_id)
+        comment_id = comment_item.get("comment_id")
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            comment_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_comment(comment_item)
+        else:
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
+
+    async def store_creator(self, creator: Dict):
+        """
+        tieba creator SQLite storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        from .tieba_store_sql import (add_new_creator,
+                                      query_creator_by_user_id,
+                                      update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)

store/weibo/weibo_store_impl.py

@@ -260,3 +260,67 @@ class WeiboJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(creator, "creators")
+
+
+class WeiboSqliteStoreImplement(AbstractStore):
+    async def store_content(self, content_item: Dict):
+        """
+        Weibo content SQLite storage implementation
+        Args:
+            content_item: content item dict
+
+        Returns:
+
+        """
+
+        from .weibo_store_sql import (add_new_content,
+                                      query_content_by_content_id,
+                                      update_content_by_content_id)
+        note_id = content_item.get("note_id")
+        note_detail: Dict = await query_content_by_content_id(content_id=note_id)
+        if not note_detail:
+            content_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_content(content_item)
+        else:
+            await update_content_by_content_id(note_id, content_item=content_item)
+
+    async def store_comment(self, comment_item: Dict):
+        """
+        Weibo comment SQLite storage implementation
+        Args:
+            comment_item: comment item dict
+
+        Returns:
+
+        """
+        from .weibo_store_sql import (add_new_comment,
+                                      query_comment_by_comment_id,
+                                      update_comment_by_comment_id)
+        comment_id = comment_item.get("comment_id")
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            comment_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_comment(comment_item)
+        else:
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
+
+    async def store_creator(self, creator: Dict):
+        """
+        Weibo creator SQLite storage implementation
+        Args:
+            creator:
+
+        Returns:
+
+        """
+
+        from .weibo_store_sql import (add_new_creator,
+                                      query_creator_by_user_id,
+                                      update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)

store/xhs/xhs_store_impl.py

@@ -255,3 +255,64 @@ class XhsJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(creator, "creator")
+
+
+class XhsSqliteStoreImplement(AbstractStore):
+    async def store_content(self, content_item: Dict):
+        """
+        Xiaohongshu content SQLite storage implementation
+        Args:
+            content_item: content item dict
+
+        Returns:
+
+        """
+        from .xhs_store_sql import (add_new_content,
+                                    query_content_by_content_id,
+                                    update_content_by_content_id)
+        note_id = content_item.get("note_id")
+        note_detail: Dict = await query_content_by_content_id(content_id=note_id)
+        if not note_detail:
+            content_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_content(content_item)
+        else:
+            await update_content_by_content_id(note_id, content_item=content_item)
+
+    async def store_comment(self, comment_item: Dict):
+        """
+        Xiaohongshu comment SQLite storage implementation
+        Args:
+            comment_item: comment item dict
+
+        Returns:
+
+        """
+        from .xhs_store_sql import (add_new_comment,
+                                    query_comment_by_comment_id,
+                                    update_comment_by_comment_id)
+        comment_id = comment_item.get("comment_id")
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            comment_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_comment(comment_item)
+        else:
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
+
+    async def store_creator(self, creator: Dict):
+        """
+        Xiaohongshu creator SQLite storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        from .xhs_store_sql import (add_new_creator, query_creator_by_user_id,
+                                    update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)

store/zhihu/zhihu_store_impl.py

@@ -254,3 +254,65 @@ class ZhihuJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(creator, "creator")
+
+
+class ZhihuSqliteStoreImplement(AbstractStore):
+    async def store_content(self, content_item: Dict):
+        """
+        Zhihu content SQLite storage implementation
+        Args:
+            content_item: content item dict
+
+        Returns:
+
+        """
+        from .zhihu_store_sql import (add_new_content,
+                                      query_content_by_content_id,
+                                      update_content_by_content_id)
+        note_id = content_item.get("note_id")
+        note_detail: Dict = await query_content_by_content_id(content_id=note_id)
+        if not note_detail:
+            content_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_content(content_item)
+        else:
+            await update_content_by_content_id(note_id, content_item=content_item)
+
+    async def store_comment(self, comment_item: Dict):
+        """
+        Zhihu comment SQLite storage implementation
+        Args:
+            comment_item: comment item dict
+
+        Returns:
+
+        """
+        from .zhihu_store_sql import (add_new_comment,
+                                      query_comment_by_comment_id,
+                                      update_comment_by_comment_id)
+        comment_id = comment_item.get("comment_id")
+        comment_detail: Dict = await query_comment_by_comment_id(comment_id=comment_id)
+        if not comment_detail:
+            comment_item["add_ts"] = utils.get_current_timestamp()
+            await add_new_comment(comment_item)
+        else:
+            await update_comment_by_comment_id(comment_id, comment_item=comment_item)
+
+    async def store_creator(self, creator: Dict):
+        """
+        Zhihu creator SQLite storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        from .zhihu_store_sql import (add_new_creator,
+                                      query_creator_by_user_id,
+                                      update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)