jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-07-23 21:00:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 将平台业务ID统一改为String并自动建表

Browse Source 
- 抖音、B站、快手、微博的帖子/视频/评论ID从BigInteger改为String，
  避免PostgreSQL下字符串写入BIGINT报错及未来ID溢出风险
- B站dynamic_id改为String，修复API返回id_str被强转int导致的精度丢失
- 知乎提取器对content_id/question_id显式str()转换
- main.py启动数据库保存模式时自动建表，无需手动--init_db
- 同步更新相关老化测试
This commit is contained in:
程序员阿江(Relakkes)
2026-07-01 23:02:13 +08:00
parent 51d4853db5
commit a06273ea6c
11 changed files with 55 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
database/models.py
View File

@@ -33,7 +33,7 @@ Base = declarative_base()
class BilibiliVideo(Base):
__tablename__ = 'bilibili_video'
id = Column(Integer, primary_key=True, comment='主键ID')
video_id = Column(BigInteger, nullable=False, index=True, unique=True, comment='视频ID')
video_id = Column(String(64), nullable=False, index=True, unique=True, comment='视频ID')
video_url = Column(Text, nullable=False, comment='视频URL')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
@@ -61,8 +61,8 @@ class BilibiliVideoComment(Base):
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
video_id = Column(BigInteger, index=True, comment='视频ID')
comment_id = Column(String(128), index=True, comment='评论ID')
video_id = Column(String(64), index=True, comment='视频ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
sub_comment_count = Column(Text, comment='子评论数')
@@ -72,7 +72,7 @@ class BilibiliVideoComment(Base):
class BilibiliUpDynamic(Base):
__tablename__ = 'bilibili_up_dynamic'
id = Column(Integer, primary_key=True, comment='主键ID')
dynamic_id = Column(BigInteger, index=True, comment='动态ID')
dynamic_id = Column(String(128), index=True, comment='动态ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_name = Column(Text, comment='用户名称(已脱敏)')
text = Column(Text, comment='动态内容')
@@ -91,7 +91,7 @@ class DouyinAweme(Base):
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
aweme_id = Column(BigInteger, index=True, comment='作品ID')
aweme_id = Column(String(255), index=True, comment='作品ID')
aweme_type = Column(Text, comment='作品类型')
title = Column(Text, comment='作品标题')
desc = Column(Text, comment='作品描述')
@@ -114,8 +114,8 @@ class DouyinAwemeComment(Base):
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
aweme_id = Column(BigInteger, index=True, comment='作品ID')
comment_id = Column(String(255), index=True, comment='评论ID')
aweme_id = Column(String(255), index=True, comment='作品ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
sub_comment_count = Column(Text, comment='子评论数')
@@ -149,7 +149,7 @@ class KuaishouVideoComment(Base):
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
comment_id = Column(String(255), index=True, comment='评论ID')
video_id = Column(String(255), index=True, comment='视频ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
@@ -162,7 +162,7 @@ class WeiboNote(Base):
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
note_id = Column(BigInteger, index=True, comment='笔记ID')
note_id = Column(String(64), index=True, comment='笔记ID')
content = Column(Text, comment='笔记内容')
create_time = Column(BigInteger, index=True, comment='创建时间戳')
create_date_time = Column(String(255), index=True, comment='创建日期时间')
@@ -179,8 +179,8 @@ class WeiboNoteComment(Base):
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
note_id = Column(BigInteger, index=True, comment='笔记ID')
comment_id = Column(String(64), index=True, comment='评论ID')
note_id = Column(String(64), index=True, comment='笔记ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
create_date_time = Column(String(255), index=True, comment='创建日期时间')