jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-02-20 22:11:11 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
be306c6f544fadce4fa7c44c7c3c287f48566b97
MediaCrawler/database/models.py
persist-1 be306c6f54 refactor(database): 重构数据库存储实现，使用SQLAlchemy ORM替代原始SQL操作 
- 删除旧的async_db.py和async_sqlite_db.py实现
- 新增SQLAlchemy ORM模型和数据库会话管理
- 统一各平台存储实现到_store_impl.py文件
- 添加数据库初始化功能支持
- 更新.gitignore和pyproject.toml依赖配置
- 优化文件存储路径和命名规范
2025-09-06 04:10:20 +08:00

434 lines
14 KiB
Python
Raw Blame History

from sqlalchemy import create_engine, Column, Integer, String, Text, BigInteger
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
class BilibiliVideo(Base):
__tablename__ = 'bilibili_video'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
video_id = Column(String, index=True)
video_type = Column(String)
title = Column(String)
desc = Column(Text)
create_time = Column(BigInteger, index=True)
liked_count = Column(String)
disliked_count = Column(String)
video_play_count = Column(String)
video_favorite_count = Column(String)
video_share_count = Column(String)
video_coin_count = Column(String)
video_danmaku = Column(String)
video_comment = Column(String)
video_url = Column(String)
video_cover_url = Column(String)
source_keyword = Column(String, default='')
class BilibiliVideoComment(Base):
__tablename__ = 'bilibili_video_comment'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
sex = Column(String)
sign = Column(String)
avatar = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
comment_id = Column(String, index=True)
video_id = Column(String, index=True)
content = Column(Text)
create_time = Column(BigInteger)
sub_comment_count = Column(String)
parent_comment_id = Column(String)
like_count = Column(String, default='0')
class BilibiliUpInfo(Base):
__tablename__ = 'bilibili_up_info'
id = Column(Integer, primary_key=True)
user_id = Column(String, index=True)
nickname = Column(String)
sex = Column(String)
sign = Column(String)
avatar = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
total_fans = Column(Integer)
total_liked = Column(Integer)
user_rank = Column(Integer)
is_official = Column(Integer)
class BilibiliContactInfo(Base):
__tablename__ = 'bilibili_contact_info'
id = Column(Integer, primary_key=True)
up_id = Column(String, index=True)
fan_id = Column(String, index=True)
up_name = Column(String)
fan_name = Column(String)
up_sign = Column(String)
fan_sign = Column(String)
up_avatar = Column(String)
fan_avatar = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
class BilibiliUpDynamic(Base):
__tablename__ = 'bilibili_up_dynamic'
id = Column(Integer, primary_key=True)
dynamic_id = Column(String, index=True)
user_id = Column(String)
user_name = Column(String)
text = Column(Text)
type = Column(String)
pub_ts = Column(BigInteger)
total_comments = Column(Integer)
total_forwards = Column(Integer)
total_liked = Column(Integer)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
class DouyinAweme(Base):
__tablename__ = 'douyin_aweme'
id = Column(Integer, primary_key=True)
user_id = Column(String)
sec_uid = Column(String)
short_user_id = Column(String)
user_unique_id = Column(String)
nickname = Column(String)
avatar = Column(String)
user_signature = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
aweme_id = Column(String, index=True)
aweme_type = Column(String)
title = Column(String)
desc = Column(Text)
create_time = Column(BigInteger, index=True)
liked_count = Column(String)
comment_count = Column(String)
share_count = Column(String)
collected_count = Column(String)
aweme_url = Column(String)
cover_url = Column(String)
video_download_url = Column(String)
music_download_url = Column(String)
note_download_url = Column(String)
source_keyword = Column(String, default='')
class DouyinAwemeComment(Base):
__tablename__ = 'douyin_aweme_comment'
id = Column(Integer, primary_key=True)
user_id = Column(String)
sec_uid = Column(String)
short_user_id = Column(String)
user_unique_id = Column(String)
nickname = Column(String)
avatar = Column(String)
user_signature = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
comment_id = Column(String, index=True)
aweme_id = Column(String, index=True)
content = Column(Text)
create_time = Column(BigInteger)
sub_comment_count = Column(String)
parent_comment_id = Column(String)
like_count = Column(String, default='0')
pictures = Column(String, default='')
class DyCreator(Base):
__tablename__ = 'dy_creator'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
desc = Column(Text)
gender = Column(String)
follows = Column(String)
fans = Column(String)
interaction = Column(String)
videos_count = Column(String)
class KuaishouVideo(Base):
__tablename__ = 'kuaishou_video'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
video_id = Column(String, index=True)
video_type = Column(String)
title = Column(String)
desc = Column(Text)
create_time = Column(BigInteger, index=True)
liked_count = Column(String)
viewd_count = Column(String)
video_url = Column(String)
video_cover_url = Column(String)
video_play_url = Column(String)
source_keyword = Column(String, default='')
class KuaishouVideoComment(Base):
__tablename__ = 'kuaishou_video_comment'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
comment_id = Column(String, index=True)
video_id = Column(String, index=True)
content = Column(Text)
create_time = Column(BigInteger)
sub_comment_count = Column(String)
class WeiboNote(Base):
__tablename__ = 'weibo_note'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
gender = Column(String)
profile_url = Column(String)
ip_location = Column(String, default='')
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
note_id = Column(String, index=True)
content = Column(Text)
create_time = Column(BigInteger, index=True)
create_date_time = Column(String, index=True)
liked_count = Column(String)
comments_count = Column(String)
shared_count = Column(String)
note_url = Column(String)
source_keyword = Column(String, default='')
class WeiboNoteComment(Base):
__tablename__ = 'weibo_note_comment'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
gender = Column(String)
profile_url = Column(String)
ip_location = Column(String, default='')
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
comment_id = Column(String, index=True)
note_id = Column(String, index=True)
content = Column(Text)
create_time = Column(BigInteger)
create_date_time = Column(String, index=True)
comment_like_count = Column(String)
sub_comment_count = Column(String)
parent_comment_id = Column(String)
class WeiboCreator(Base):
__tablename__ = 'weibo_creator'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
desc = Column(Text)
gender = Column(String)
follows = Column(String)
fans = Column(String)
tag_list = Column(String)
class XhsCreator(Base):
__tablename__ = 'xhs_creator'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
desc = Column(Text)
gender = Column(String)
follows = Column(String)
fans = Column(String)
interaction = Column(String)
tag_list = Column(String)
class XhsNote(Base):
__tablename__ = 'xhs_note'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
note_id = Column(String, index=True)
type = Column(String)
title = Column(String)
desc = Column(Text)
video_url = Column(String)
time = Column(BigInteger, index=True)
last_update_time = Column(BigInteger)
liked_count = Column(String)
collected_count = Column(String)
comment_count = Column(String)
share_count = Column(String)
image_list = Column(Text)
tag_list = Column(Text)
note_url = Column(String)
source_keyword = Column(String, default='')
xsec_token = Column(String)
class XhsNoteComment(Base):
__tablename__ = 'xhs_note_comment'
id = Column(Integer, primary_key=True)
user_id = Column(String)
nickname = Column(String)
avatar = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
comment_id = Column(String, index=True)
create_time = Column(BigInteger, index=True)
note_id = Column(String)
content = Column(Text)
sub_comment_count = Column(Integer)
pictures = Column(Text)
parent_comment_id = Column(String)
like_count = Column(String)
class TiebaNote(Base):
__tablename__ = 'tieba_note'
id = Column(Integer, primary_key=True)
note_id = Column(String, index=True)
title = Column(String)
desc = Column(Text)
note_url = Column(String)
publish_time = Column(String, index=True)
user_link = Column(String, default='')
user_nickname = Column(String, default='')
user_avatar = Column(String, default='')
tieba_id = Column(String, default='')
tieba_name = Column(String)
tieba_link = Column(String)
total_replay_num = Column(Integer, default=0)
total_replay_page = Column(Integer, default=0)
ip_location = Column(String, default='')
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
source_keyword = Column(String, default='')
class TiebaComment(Base):
__tablename__ = 'tieba_comment'
id = Column(Integer, primary_key=True)
comment_id = Column(String, index=True)
parent_comment_id = Column(String, default='')
content = Column(Text)
user_link = Column(String, default='')
user_nickname = Column(String, default='')
user_avatar = Column(String, default='')
tieba_id = Column(String, default='')
tieba_name = Column(String)
tieba_link = Column(String)
publish_time = Column(String, index=True)
ip_location = Column(String, default='')
sub_comment_count = Column(Integer, default=0)
note_id = Column(String, index=True)
note_url = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
class TiebaCreator(Base):
__tablename__ = 'tieba_creator'
id = Column(Integer, primary_key=True)
user_id = Column(String)
user_name = Column(String)
nickname = Column(String)
avatar = Column(String)
ip_location = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
gender = Column(String)
follows = Column(String)
fans = Column(String)
registration_duration = Column(String)
class ZhihuContent(Base):
__tablename__ = 'zhihu_content'
id = Column(Integer, primary_key=True)
content_id = Column(String, index=True)
content_type = Column(String)
content_text = Column(Text)
content_url = Column(String)
question_id = Column(String)
title = Column(String)
desc = Column(Text)
created_time = Column(String, index=True)
updated_time = Column(String)
voteup_count = Column(Integer, default=0)
comment_count = Column(Integer, default=0)
source_keyword = Column(String)
user_id = Column(String)
user_link = Column(String)
user_nickname = Column(String)
user_avatar = Column(String)
user_url_token = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
# persist-1<persist1@126.com>
# 原因：修复 ORM 模型定义错误，确保与数据库表结构一致。
# 副作用：无
# 回滚策略：还原此行
class ZhihuComment(Base):
__tablename__ = 'zhihu_comment'
id = Column(Integer, primary_key=True)
comment_id = Column(String, index=True)
parent_comment_id = Column(String)
content = Column(Text)
publish_time = Column(String, index=True)
ip_location = Column(String)
sub_comment_count = Column(Integer, default=0)
like_count = Column(Integer, default=0)
dislike_count = Column(Integer, default=0)
content_id = Column(String, index=True)
content_type = Column(String)
user_id = Column(String)
user_link = Column(String)
user_nickname = Column(String)
user_avatar = Column(String)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
class ZhihuCreator(Base):
__tablename__ = 'zhihu_creator'
id = Column(Integer, primary_key=True)
user_id = Column(String, unique=True, index=True)
user_link = Column(String)
user_nickname = Column(String)
user_avatar = Column(String)
url_token = Column(String)
gender = Column(String)
ip_location = Column(String)
follows = Column(Integer, default=0)
fans = Column(Integer, default=0)
anwser_count = Column(Integer, default=0)
video_count = Column(Integer, default=0)
question_count = Column(Integer, default=0)
article_count = Column(Integer, default=0)
column_count = Column(Integer, default=0)
get_voteup_count = Column(Integer, default=0)
add_ts = Column(BigInteger)
last_modify_ts = Column(BigInteger)
Reference in New Issue View Git Blame Copy Permalink