From 12980224106ddf5102498ad8e0c402b265beff41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=B0=E5=AE=9A=E4=B8=8D=E7=A6=BB=E6=89=8B?= <12640033+msz-006@user.noreply.gitee.com> Date: Mon, 14 Jul 2025 03:51:08 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E6=9B=B4=E6=96=B0=E5=90=84?= =?UTF-8?q?=E5=B9=B3=E5=8F=B0store=E6=A8=A1=E5=9D=97=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E4=BB=A5=E6=94=AF=E6=8C=81SQLite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 更新 store/bilibili/__init__.py 文件，导入SQLite存储实现类和相关模块 - 更新 store/douyin/__init__.py 文件，集成抖音平台的SQLite数据存储接口 - 更新 store/kuaishou/__init__.py 文件，添加快手平台SQLite存储模块的导入声明 - 更新 store/tieba/__init__.py 文件，引入贴吧平台SQLite数据库操作模块 - 更新 store/weibo/__init__.py 文件，整合微博平台SQLite存储功能模块 - 更新 store/xhs/__init__.py 文件，导入小红书平台SQLite数据存储实现 - 更新 store/zhihu/__init__.py 文件，集成知乎平台SQLite数据库存储模块 --- store/bilibili/__init__.py | 3 ++- store/douyin/__init__.py | 3 ++- store/kuaishou/__init__.py | 5 +++-- store/tieba/__init__.py | 3 ++- store/weibo/__init__.py | 3 ++- store/xhs/__init__.py | 5 +++-- store/zhihu/__init__.py | 8 +++++--- 7 files changed, 19 insertions(+), 11 deletions(-) diff --git a/store/bilibili/__init__.py b/store/bilibili/__init__.py index 4c83079..5160aa2 100644 --- a/store/bilibili/__init__.py +++ b/store/bilibili/__init__.py @@ -28,6 +28,7 @@ class BiliStoreFactory: "csv": BiliCsvStoreImplement, "db": BiliDbStoreImplement, "json": BiliJsonStoreImplement, + "sqlite": BiliSqliteStoreImplement, } @staticmethod @@ -35,7 +36,7 @@ class BiliStoreFactory: store_class = BiliStoreFactory.STORES.get(config.SAVE_DATA_OPTION) if not store_class: raise ValueError( - "[BiliStoreFactory.create_store] Invalid save option only supported csv or db or json ..." + "[BiliStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ..." ) return store_class() diff --git a/store/douyin/__init__.py b/store/douyin/__init__.py index 3c62288..e4958e8 100644 --- a/store/douyin/__init__.py +++ b/store/douyin/__init__.py @@ -26,6 +26,7 @@ class DouyinStoreFactory: "csv": DouyinCsvStoreImplement, "db": DouyinDbStoreImplement, "json": DouyinJsonStoreImplement, + "sqlite": DouyinSqliteStoreImplement } @staticmethod @@ -33,7 +34,7 @@ class DouyinStoreFactory: store_class = DouyinStoreFactory.STORES.get(config.SAVE_DATA_OPTION) if not store_class: raise ValueError( - "[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json ..." + "[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ..." ) return store_class() diff --git a/store/kuaishou/__init__.py b/store/kuaishou/__init__.py index 2c2e012..3ce1089 100644 --- a/store/kuaishou/__init__.py +++ b/store/kuaishou/__init__.py @@ -25,7 +25,8 @@ class KuaishouStoreFactory: STORES = { "csv": KuaishouCsvStoreImplement, "db": KuaishouDbStoreImplement, - "json": KuaishouJsonStoreImplement + "json": KuaishouJsonStoreImplement, + "sqlite": KuaishouSqliteStoreImplement } @staticmethod @@ -33,7 +34,7 @@ class KuaishouStoreFactory: store_class = KuaishouStoreFactory.STORES.get(config.SAVE_DATA_OPTION) if not store_class: raise ValueError( - "[KuaishouStoreFactory.create_store] Invalid save option only supported csv or db or json ...") + "[KuaishouStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...") return store_class() diff --git a/store/tieba/__init__.py b/store/tieba/__init__.py index 1bc486f..e928d85 100644 --- a/store/tieba/__init__.py +++ b/store/tieba/__init__.py @@ -23,7 +23,8 @@ class TieBaStoreFactory: STORES = { "csv": TieBaCsvStoreImplement, "db": TieBaDbStoreImplement, - "json": TieBaJsonStoreImplement + "json": TieBaJsonStoreImplement, + "sqlite": TieBaSqliteStoreImplement } @staticmethod diff --git a/store/weibo/__init__.py b/store/weibo/__init__.py index 75bcf10..449e3dd 100644 --- a/store/weibo/__init__.py +++ b/store/weibo/__init__.py @@ -28,6 +28,7 @@ class WeibostoreFactory: "csv": WeiboCsvStoreImplement, "db": WeiboDbStoreImplement, "json": WeiboJsonStoreImplement, + "sqlite": WeiboSqliteStoreImplement, } @staticmethod @@ -35,7 +36,7 @@ class WeibostoreFactory: store_class = WeibostoreFactory.STORES.get(config.SAVE_DATA_OPTION) if not store_class: raise ValueError( - "[WeibotoreFactory.create_store] Invalid save option only supported csv or db or json ...") + "[WeibotoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...") return store_class() diff --git a/store/xhs/__init__.py b/store/xhs/__init__.py index 936bd25..f57b684 100644 --- a/store/xhs/__init__.py +++ b/store/xhs/__init__.py @@ -27,14 +27,15 @@ class XhsStoreFactory: STORES = { "csv": XhsCsvStoreImplement, "db": XhsDbStoreImplement, - "json": XhsJsonStoreImplement + "json": XhsJsonStoreImplement, + "sqlite": XhsSqliteStoreImplement } @staticmethod def create_store() -> AbstractStore: store_class = XhsStoreFactory.STORES.get(config.SAVE_DATA_OPTION) if not store_class: - raise ValueError("[XhsStoreFactory.create_store] Invalid save option only supported csv or db or json ...") + raise ValueError("[XhsStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...") return store_class() diff --git a/store/zhihu/__init__.py b/store/zhihu/__init__.py index 6a47c73..54991b0 100644 --- a/store/zhihu/__init__.py +++ b/store/zhihu/__init__.py @@ -17,7 +17,8 @@ from base.base_crawler import AbstractStore from model.m_zhihu import ZhihuComment, ZhihuContent, ZhihuCreator from store.zhihu.zhihu_store_impl import (ZhihuCsvStoreImplement, ZhihuDbStoreImplement, - ZhihuJsonStoreImplement) + ZhihuJsonStoreImplement, + ZhihuSqliteStoreImplement) from tools import utils from var import source_keyword_var @@ -26,14 +27,15 @@ class ZhihuStoreFactory: STORES = { "csv": ZhihuCsvStoreImplement, "db": ZhihuDbStoreImplement, - "json": ZhihuJsonStoreImplement + "json": ZhihuJsonStoreImplement, + "sqlite": ZhihuSqliteStoreImplement } @staticmethod def create_store() -> AbstractStore: store_class = ZhihuStoreFactory.STORES.get(config.SAVE_DATA_OPTION) if not store_class: - raise ValueError("[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json ...") + raise ValueError("[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...") return store_class() async def batch_update_zhihu_contents(contents: List[ZhihuContent]):