refactor(database): 重构数据库存储实现,使用SQLAlchemy ORM替代原始SQL操作

- 删除旧的async_db.py和async_sqlite_db.py实现
- 新增SQLAlchemy ORM模型和数据库会话管理
- 统一各平台存储实现到_store_impl.py文件
- 添加数据库初始化功能支持
- 更新.gitignore和pyproject.toml依赖配置
- 优化文件存储路径和命名规范
This commit is contained in:
persist-1
2025-09-06 04:10:20 +08:00
parent fa5f07e9ee
commit be306c6f54
46 changed files with 3362 additions and 6117 deletions

View File

@@ -71,7 +71,7 @@ USER_DATA_DIR = "%s_user_data_dir" # %s will be replaced by platform name
START_PAGE = 1
# 爬取视频/帖子的数量控制
CRAWLER_MAX_NOTES_COUNT = 200
CRAWLER_MAX_NOTES_COUNT = 15
# 并发爬虫数量控制
MAX_CONCURRENCY_NUM = 1

View File

@@ -18,6 +18,14 @@ MYSQL_DB_HOST = os.getenv("MYSQL_DB_HOST", "localhost")
MYSQL_DB_PORT = os.getenv("MYSQL_DB_PORT", 3306)
MYSQL_DB_NAME = os.getenv("MYSQL_DB_NAME", "media_crawler")
mysql_db_config = {
"user": MYSQL_DB_USER,
"password": MYSQL_DB_PWD,
"host": MYSQL_DB_HOST,
"port": MYSQL_DB_PORT,
"db_name": MYSQL_DB_NAME,
}
# redis config
REDIS_DB_HOST = "127.0.0.1" # your redis host
@@ -30,4 +38,8 @@ CACHE_TYPE_REDIS = "redis"
CACHE_TYPE_MEMORY = "memory"
# sqlite config
SQLITE_DB_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "schema", "sqlite_tables.db")
SQLITE_DB_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "database", "sqlite_tables.db")
sqlite_db_config = {
"db_path": SQLITE_DB_PATH
}