mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-03-06 14:10:46 +08:00
refactor(database): 重构数据库存储实现，使用SQLAlchemy ORM替代原始SQL操作
- 删除旧的async_db.py和async_sqlite_db.py实现 - 新增SQLAlchemy ORM模型和数据库会话管理 - 统一各平台存储实现到_store_impl.py文件 - 添加数据库初始化功能支持 - 更新.gitignore和pyproject.toml依赖配置 - 优化文件存储路径和命名规范
This commit is contained in:
persist-1
434
database/models.py
Normal file
434
database/models.py
Normal file
@@ -0,0 +1,434 @@
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Text, BigInteger
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
class BilibiliVideo(Base):
|
||||
__tablename__ = 'bilibili_video'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
video_id = Column(String, index=True)
|
||||
video_type = Column(String)
|
||||
title = Column(String)
|
||||
desc = Column(Text)
|
||||
create_time = Column(BigInteger, index=True)
|
||||
liked_count = Column(String)
|
||||
disliked_count = Column(String)
|
||||
video_play_count = Column(String)
|
||||
video_favorite_count = Column(String)
|
||||
video_share_count = Column(String)
|
||||
video_coin_count = Column(String)
|
||||
video_danmaku = Column(String)
|
||||
video_comment = Column(String)
|
||||
video_url = Column(String)
|
||||
video_cover_url = Column(String)
|
||||
source_keyword = Column(String, default='')
|
||||
|
||||
class BilibiliVideoComment(Base):
|
||||
__tablename__ = 'bilibili_video_comment'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
sex = Column(String)
|
||||
sign = Column(String)
|
||||
avatar = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
comment_id = Column(String, index=True)
|
||||
video_id = Column(String, index=True)
|
||||
content = Column(Text)
|
||||
create_time = Column(BigInteger)
|
||||
sub_comment_count = Column(String)
|
||||
parent_comment_id = Column(String)
|
||||
like_count = Column(String, default='0')
|
||||
|
||||
class BilibiliUpInfo(Base):
|
||||
__tablename__ = 'bilibili_up_info'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String, index=True)
|
||||
nickname = Column(String)
|
||||
sex = Column(String)
|
||||
sign = Column(String)
|
||||
avatar = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
total_fans = Column(Integer)
|
||||
total_liked = Column(Integer)
|
||||
user_rank = Column(Integer)
|
||||
is_official = Column(Integer)
|
||||
|
||||
class BilibiliContactInfo(Base):
|
||||
__tablename__ = 'bilibili_contact_info'
|
||||
id = Column(Integer, primary_key=True)
|
||||
up_id = Column(String, index=True)
|
||||
fan_id = Column(String, index=True)
|
||||
up_name = Column(String)
|
||||
fan_name = Column(String)
|
||||
up_sign = Column(String)
|
||||
fan_sign = Column(String)
|
||||
up_avatar = Column(String)
|
||||
fan_avatar = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
|
||||
class BilibiliUpDynamic(Base):
|
||||
__tablename__ = 'bilibili_up_dynamic'
|
||||
id = Column(Integer, primary_key=True)
|
||||
dynamic_id = Column(String, index=True)
|
||||
user_id = Column(String)
|
||||
user_name = Column(String)
|
||||
text = Column(Text)
|
||||
type = Column(String)
|
||||
pub_ts = Column(BigInteger)
|
||||
total_comments = Column(Integer)
|
||||
total_forwards = Column(Integer)
|
||||
total_liked = Column(Integer)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
|
||||
class DouyinAweme(Base):
|
||||
__tablename__ = 'douyin_aweme'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
sec_uid = Column(String)
|
||||
short_user_id = Column(String)
|
||||
user_unique_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
user_signature = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
aweme_id = Column(String, index=True)
|
||||
aweme_type = Column(String)
|
||||
title = Column(String)
|
||||
desc = Column(Text)
|
||||
create_time = Column(BigInteger, index=True)
|
||||
liked_count = Column(String)
|
||||
comment_count = Column(String)
|
||||
share_count = Column(String)
|
||||
collected_count = Column(String)
|
||||
aweme_url = Column(String)
|
||||
cover_url = Column(String)
|
||||
video_download_url = Column(String)
|
||||
music_download_url = Column(String)
|
||||
note_download_url = Column(String)
|
||||
source_keyword = Column(String, default='')
|
||||
|
||||
class DouyinAwemeComment(Base):
|
||||
__tablename__ = 'douyin_aweme_comment'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
sec_uid = Column(String)
|
||||
short_user_id = Column(String)
|
||||
user_unique_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
user_signature = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
comment_id = Column(String, index=True)
|
||||
aweme_id = Column(String, index=True)
|
||||
content = Column(Text)
|
||||
create_time = Column(BigInteger)
|
||||
sub_comment_count = Column(String)
|
||||
parent_comment_id = Column(String)
|
||||
like_count = Column(String, default='0')
|
||||
pictures = Column(String, default='')
|
||||
|
||||
class DyCreator(Base):
|
||||
__tablename__ = 'dy_creator'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
desc = Column(Text)
|
||||
gender = Column(String)
|
||||
follows = Column(String)
|
||||
fans = Column(String)
|
||||
interaction = Column(String)
|
||||
videos_count = Column(String)
|
||||
|
||||
class KuaishouVideo(Base):
|
||||
__tablename__ = 'kuaishou_video'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
video_id = Column(String, index=True)
|
||||
video_type = Column(String)
|
||||
title = Column(String)
|
||||
desc = Column(Text)
|
||||
create_time = Column(BigInteger, index=True)
|
||||
liked_count = Column(String)
|
||||
viewd_count = Column(String)
|
||||
video_url = Column(String)
|
||||
video_cover_url = Column(String)
|
||||
video_play_url = Column(String)
|
||||
source_keyword = Column(String, default='')
|
||||
|
||||
class KuaishouVideoComment(Base):
|
||||
__tablename__ = 'kuaishou_video_comment'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
comment_id = Column(String, index=True)
|
||||
video_id = Column(String, index=True)
|
||||
content = Column(Text)
|
||||
create_time = Column(BigInteger)
|
||||
sub_comment_count = Column(String)
|
||||
|
||||
class WeiboNote(Base):
|
||||
__tablename__ = 'weibo_note'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
gender = Column(String)
|
||||
profile_url = Column(String)
|
||||
ip_location = Column(String, default='')
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
note_id = Column(String, index=True)
|
||||
content = Column(Text)
|
||||
create_time = Column(BigInteger, index=True)
|
||||
create_date_time = Column(String, index=True)
|
||||
liked_count = Column(String)
|
||||
comments_count = Column(String)
|
||||
shared_count = Column(String)
|
||||
note_url = Column(String)
|
||||
source_keyword = Column(String, default='')
|
||||
|
||||
class WeiboNoteComment(Base):
|
||||
__tablename__ = 'weibo_note_comment'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
gender = Column(String)
|
||||
profile_url = Column(String)
|
||||
ip_location = Column(String, default='')
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
comment_id = Column(String, index=True)
|
||||
note_id = Column(String, index=True)
|
||||
content = Column(Text)
|
||||
create_time = Column(BigInteger)
|
||||
create_date_time = Column(String, index=True)
|
||||
comment_like_count = Column(String)
|
||||
sub_comment_count = Column(String)
|
||||
parent_comment_id = Column(String)
|
||||
|
||||
class WeiboCreator(Base):
|
||||
__tablename__ = 'weibo_creator'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
desc = Column(Text)
|
||||
gender = Column(String)
|
||||
follows = Column(String)
|
||||
fans = Column(String)
|
||||
tag_list = Column(String)
|
||||
|
||||
class XhsCreator(Base):
|
||||
__tablename__ = 'xhs_creator'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
desc = Column(Text)
|
||||
gender = Column(String)
|
||||
follows = Column(String)
|
||||
fans = Column(String)
|
||||
interaction = Column(String)
|
||||
tag_list = Column(String)
|
||||
|
||||
class XhsNote(Base):
|
||||
__tablename__ = 'xhs_note'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
note_id = Column(String, index=True)
|
||||
type = Column(String)
|
||||
title = Column(String)
|
||||
desc = Column(Text)
|
||||
video_url = Column(String)
|
||||
time = Column(BigInteger, index=True)
|
||||
last_update_time = Column(BigInteger)
|
||||
liked_count = Column(String)
|
||||
collected_count = Column(String)
|
||||
comment_count = Column(String)
|
||||
share_count = Column(String)
|
||||
image_list = Column(Text)
|
||||
tag_list = Column(Text)
|
||||
note_url = Column(String)
|
||||
source_keyword = Column(String, default='')
|
||||
xsec_token = Column(String)
|
||||
|
||||
class XhsNoteComment(Base):
|
||||
__tablename__ = 'xhs_note_comment'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
comment_id = Column(String, index=True)
|
||||
create_time = Column(BigInteger, index=True)
|
||||
note_id = Column(String)
|
||||
content = Column(Text)
|
||||
sub_comment_count = Column(Integer)
|
||||
pictures = Column(Text)
|
||||
parent_comment_id = Column(String)
|
||||
like_count = Column(String)
|
||||
|
||||
class TiebaNote(Base):
|
||||
__tablename__ = 'tieba_note'
|
||||
id = Column(Integer, primary_key=True)
|
||||
note_id = Column(String, index=True)
|
||||
title = Column(String)
|
||||
desc = Column(Text)
|
||||
note_url = Column(String)
|
||||
publish_time = Column(String, index=True)
|
||||
user_link = Column(String, default='')
|
||||
user_nickname = Column(String, default='')
|
||||
user_avatar = Column(String, default='')
|
||||
tieba_id = Column(String, default='')
|
||||
tieba_name = Column(String)
|
||||
tieba_link = Column(String)
|
||||
total_replay_num = Column(Integer, default=0)
|
||||
total_replay_page = Column(Integer, default=0)
|
||||
ip_location = Column(String, default='')
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
source_keyword = Column(String, default='')
|
||||
|
||||
class TiebaComment(Base):
|
||||
__tablename__ = 'tieba_comment'
|
||||
id = Column(Integer, primary_key=True)
|
||||
comment_id = Column(String, index=True)
|
||||
parent_comment_id = Column(String, default='')
|
||||
content = Column(Text)
|
||||
user_link = Column(String, default='')
|
||||
user_nickname = Column(String, default='')
|
||||
user_avatar = Column(String, default='')
|
||||
tieba_id = Column(String, default='')
|
||||
tieba_name = Column(String)
|
||||
tieba_link = Column(String)
|
||||
publish_time = Column(String, index=True)
|
||||
ip_location = Column(String, default='')
|
||||
sub_comment_count = Column(Integer, default=0)
|
||||
note_id = Column(String, index=True)
|
||||
note_url = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
|
||||
class TiebaCreator(Base):
|
||||
__tablename__ = 'tieba_creator'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String)
|
||||
user_name = Column(String)
|
||||
nickname = Column(String)
|
||||
avatar = Column(String)
|
||||
ip_location = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
gender = Column(String)
|
||||
follows = Column(String)
|
||||
fans = Column(String)
|
||||
registration_duration = Column(String)
|
||||
|
||||
class ZhihuContent(Base):
|
||||
__tablename__ = 'zhihu_content'
|
||||
id = Column(Integer, primary_key=True)
|
||||
content_id = Column(String, index=True)
|
||||
content_type = Column(String)
|
||||
content_text = Column(Text)
|
||||
content_url = Column(String)
|
||||
question_id = Column(String)
|
||||
title = Column(String)
|
||||
desc = Column(Text)
|
||||
created_time = Column(String, index=True)
|
||||
updated_time = Column(String)
|
||||
voteup_count = Column(Integer, default=0)
|
||||
comment_count = Column(Integer, default=0)
|
||||
source_keyword = Column(String)
|
||||
user_id = Column(String)
|
||||
user_link = Column(String)
|
||||
user_nickname = Column(String)
|
||||
user_avatar = Column(String)
|
||||
user_url_token = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
|
||||
# persist-1<persist1@126.com>
|
||||
# 原因：修复 ORM 模型定义错误，确保与数据库表结构一致。
|
||||
# 副作用：无
|
||||
# 回滚策略：还原此行
|
||||
|
||||
class ZhihuComment(Base):
|
||||
__tablename__ = 'zhihu_comment'
|
||||
id = Column(Integer, primary_key=True)
|
||||
comment_id = Column(String, index=True)
|
||||
parent_comment_id = Column(String)
|
||||
content = Column(Text)
|
||||
publish_time = Column(String, index=True)
|
||||
ip_location = Column(String)
|
||||
sub_comment_count = Column(Integer, default=0)
|
||||
like_count = Column(Integer, default=0)
|
||||
dislike_count = Column(Integer, default=0)
|
||||
content_id = Column(String, index=True)
|
||||
content_type = Column(String)
|
||||
user_id = Column(String)
|
||||
user_link = Column(String)
|
||||
user_nickname = Column(String)
|
||||
user_avatar = Column(String)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
|
||||
class ZhihuCreator(Base):
|
||||
__tablename__ = 'zhihu_creator'
|
||||
id = Column(Integer, primary_key=True)
|
||||
user_id = Column(String, unique=True, index=True)
|
||||
user_link = Column(String)
|
||||
user_nickname = Column(String)
|
||||
user_avatar = Column(String)
|
||||
url_token = Column(String)
|
||||
gender = Column(String)
|
||||
ip_location = Column(String)
|
||||
follows = Column(Integer, default=0)
|
||||
fans = Column(Integer, default=0)
|
||||
anwser_count = Column(Integer, default=0)
|
||||
video_count = Column(Integer, default=0)
|
||||
question_count = Column(Integer, default=0)
|
||||
article_count = Column(Integer, default=0)
|
||||
column_count = Column(Integer, default=0)
|
||||
get_voteup_count = Column(Integer, default=0)
|
||||
add_ts = Column(BigInteger)
|
||||
last_modify_ts = Column(BigInteger)
|
||||
Reference in New Issue
Block a user