mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-05 01:17:26 +08:00
refactor: 数据存储重构,分离不同类型的存储实现
This commit is contained in:
@@ -20,6 +20,9 @@ HEADLESS = True
|
||||
# 是否保存登录状态
|
||||
SAVE_LOGIN_STATE = True
|
||||
|
||||
# 数据保存类型选项配置,支持三种类型:csv、db、json
|
||||
SAVE_DATA_OPTION = "csv" # csv or db or json
|
||||
|
||||
# 用户浏览器缓存的浏览器文件配置
|
||||
USER_DATA_DIR = "%s_user_data_dir" # %s will be replaced by platform name
|
||||
|
||||
@@ -54,7 +57,10 @@ DY_SPECIFIED_ID_LIST = [
|
||||
]
|
||||
|
||||
# 指定快手平台需要爬取的ID列表
|
||||
KS_SPECIFIED_ID_LIST = []
|
||||
KS_SPECIFIED_ID_LIST = [
|
||||
"3xf8enb8dbj6uig",
|
||||
"3x6zz972bchmvqe"
|
||||
]
|
||||
|
||||
# 指定B站平台需要爬取的视频bvid列表
|
||||
BILI_SPECIFIED_ID_LIST = [
|
||||
|
||||
@@ -7,6 +7,3 @@ REDIS_DB_PWD = os.getenv("REDIS_DB_PWD", "123456") # your redis password
|
||||
# mysql config
|
||||
RELATION_DB_PWD = os.getenv("RELATION_DB_PWD", "123456") # your relation db password
|
||||
RELATION_DB_URL = f"mysql://root:{RELATION_DB_PWD}@localhost:3306/media_crawler"
|
||||
|
||||
# save data to database option
|
||||
IS_SAVED_DATABASED = False # if you want to save data to database, set True
|
||||
|
||||
Reference in New Issue
Block a user