mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-03-08 07:00:46 +08:00
refactor: 更新各平台store模块初始化以支持SQLite
- 更新 store/bilibili/__init__.py 文件,导入SQLite存储实现类和相关模块 - 更新 store/douyin/__init__.py 文件,集成抖音平台的SQLite数据存储接口 - 更新 store/kuaishou/__init__.py 文件,添加快手平台SQLite存储模块的导入声明 - 更新 store/tieba/__init__.py 文件,引入贴吧平台SQLite数据库操作模块 - 更新 store/weibo/__init__.py 文件,整合微博平台SQLite存储功能模块 - 更新 store/xhs/__init__.py 文件,导入小红书平台SQLite数据存储实现 - 更新 store/zhihu/__init__.py 文件,集成知乎平台SQLite数据库存储模块
This commit is contained in:
@@ -28,6 +28,7 @@ class BiliStoreFactory:
|
||||
"csv": BiliCsvStoreImplement,
|
||||
"db": BiliDbStoreImplement,
|
||||
"json": BiliJsonStoreImplement,
|
||||
"sqlite": BiliSqliteStoreImplement,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -35,7 +36,7 @@ class BiliStoreFactory:
|
||||
store_class = BiliStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
|
||||
if not store_class:
|
||||
raise ValueError(
|
||||
"[BiliStoreFactory.create_store] Invalid save option only supported csv or db or json ..."
|
||||
"[BiliStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ..."
|
||||
)
|
||||
return store_class()
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ class DouyinStoreFactory:
|
||||
"csv": DouyinCsvStoreImplement,
|
||||
"db": DouyinDbStoreImplement,
|
||||
"json": DouyinJsonStoreImplement,
|
||||
"sqlite": DouyinSqliteStoreImplement
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -33,7 +34,7 @@ class DouyinStoreFactory:
|
||||
store_class = DouyinStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
|
||||
if not store_class:
|
||||
raise ValueError(
|
||||
"[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json ..."
|
||||
"[DouyinStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ..."
|
||||
)
|
||||
return store_class()
|
||||
|
||||
|
||||
@@ -25,7 +25,8 @@ class KuaishouStoreFactory:
|
||||
STORES = {
|
||||
"csv": KuaishouCsvStoreImplement,
|
||||
"db": KuaishouDbStoreImplement,
|
||||
"json": KuaishouJsonStoreImplement
|
||||
"json": KuaishouJsonStoreImplement,
|
||||
"sqlite": KuaishouSqliteStoreImplement
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -33,7 +34,7 @@ class KuaishouStoreFactory:
|
||||
store_class = KuaishouStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
|
||||
if not store_class:
|
||||
raise ValueError(
|
||||
"[KuaishouStoreFactory.create_store] Invalid save option only supported csv or db or json ...")
|
||||
"[KuaishouStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...")
|
||||
return store_class()
|
||||
|
||||
|
||||
|
||||
@@ -23,7 +23,8 @@ class TieBaStoreFactory:
|
||||
STORES = {
|
||||
"csv": TieBaCsvStoreImplement,
|
||||
"db": TieBaDbStoreImplement,
|
||||
"json": TieBaJsonStoreImplement
|
||||
"json": TieBaJsonStoreImplement,
|
||||
"sqlite": TieBaSqliteStoreImplement
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -28,6 +28,7 @@ class WeibostoreFactory:
|
||||
"csv": WeiboCsvStoreImplement,
|
||||
"db": WeiboDbStoreImplement,
|
||||
"json": WeiboJsonStoreImplement,
|
||||
"sqlite": WeiboSqliteStoreImplement,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -35,7 +36,7 @@ class WeibostoreFactory:
|
||||
store_class = WeibostoreFactory.STORES.get(config.SAVE_DATA_OPTION)
|
||||
if not store_class:
|
||||
raise ValueError(
|
||||
"[WeibotoreFactory.create_store] Invalid save option only supported csv or db or json ...")
|
||||
"[WeibotoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...")
|
||||
return store_class()
|
||||
|
||||
|
||||
|
||||
@@ -27,14 +27,15 @@ class XhsStoreFactory:
|
||||
STORES = {
|
||||
"csv": XhsCsvStoreImplement,
|
||||
"db": XhsDbStoreImplement,
|
||||
"json": XhsJsonStoreImplement
|
||||
"json": XhsJsonStoreImplement,
|
||||
"sqlite": XhsSqliteStoreImplement
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def create_store() -> AbstractStore:
|
||||
store_class = XhsStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
|
||||
if not store_class:
|
||||
raise ValueError("[XhsStoreFactory.create_store] Invalid save option only supported csv or db or json ...")
|
||||
raise ValueError("[XhsStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...")
|
||||
return store_class()
|
||||
|
||||
|
||||
|
||||
@@ -17,7 +17,8 @@ from base.base_crawler import AbstractStore
|
||||
from model.m_zhihu import ZhihuComment, ZhihuContent, ZhihuCreator
|
||||
from store.zhihu.zhihu_store_impl import (ZhihuCsvStoreImplement,
|
||||
ZhihuDbStoreImplement,
|
||||
ZhihuJsonStoreImplement)
|
||||
ZhihuJsonStoreImplement,
|
||||
ZhihuSqliteStoreImplement)
|
||||
from tools import utils
|
||||
from var import source_keyword_var
|
||||
|
||||
@@ -26,14 +27,15 @@ class ZhihuStoreFactory:
|
||||
STORES = {
|
||||
"csv": ZhihuCsvStoreImplement,
|
||||
"db": ZhihuDbStoreImplement,
|
||||
"json": ZhihuJsonStoreImplement
|
||||
"json": ZhihuJsonStoreImplement,
|
||||
"sqlite": ZhihuSqliteStoreImplement
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def create_store() -> AbstractStore:
|
||||
store_class = ZhihuStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
|
||||
if not store_class:
|
||||
raise ValueError("[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json ...")
|
||||
raise ValueError("[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite ...")
|
||||
return store_class()
|
||||
|
||||
async def batch_update_zhihu_contents(contents: List[ZhihuContent]):
|
||||
|
||||
Reference in New Issue
Block a user