mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-07-28 23:40:30 +08:00
fix: 将平台业务ID统一改为String并自动建表
- 抖音、B站、快手、微博的帖子/视频/评论ID从BigInteger改为String, 避免PostgreSQL下字符串写入BIGINT报错及未来ID溢出风险 - B站dynamic_id改为String,修复API返回id_str被强转int导致的精度丢失 - 知乎提取器对content_id/question_id显式str()转换 - main.py启动数据库保存模式时自动建表,无需手动--init_db - 同步更新相关老化测试
This commit is contained in:
@@ -128,8 +128,7 @@ class BiliDbStoreImplement(AbstractStore):
|
||||
Args:
|
||||
content_item: content item dict
|
||||
"""
|
||||
video_id = int(content_item.get("video_id"))
|
||||
content_item["video_id"] = video_id
|
||||
video_id = content_item.get("video_id")
|
||||
content_item["liked_count"] = int(content_item.get("liked_count", 0) or 0)
|
||||
content_item["create_time"] = int(content_item.get("create_time", 0) or 0)
|
||||
|
||||
@@ -154,9 +153,7 @@ class BiliDbStoreImplement(AbstractStore):
|
||||
Args:
|
||||
comment_item: comment item dict
|
||||
"""
|
||||
comment_id = int(comment_item.get("comment_id"))
|
||||
comment_item["comment_id"] = comment_id
|
||||
comment_item["video_id"] = int(comment_item.get("video_id", 0) or 0)
|
||||
comment_id = comment_item.get("comment_id")
|
||||
comment_item["create_time"] = int(comment_item.get("create_time", 0) or 0)
|
||||
comment_item["like_count"] = str(comment_item.get("like_count", "0"))
|
||||
comment_item["sub_comment_count"] = str(comment_item.get("sub_comment_count", "0"))
|
||||
@@ -191,8 +188,7 @@ class BiliDbStoreImplement(AbstractStore):
|
||||
Args:
|
||||
dynamic_item: dynamic item dict
|
||||
"""
|
||||
dynamic_id = int(dynamic_item.get("dynamic_id"))
|
||||
dynamic_item["dynamic_id"] = dynamic_id
|
||||
dynamic_id = dynamic_item.get("dynamic_id")
|
||||
|
||||
async with get_session() as session:
|
||||
result = await session.execute(select(BilibiliUpDynamic).where(BilibiliUpDynamic.dynamic_id == dynamic_id))
|
||||
|
||||
@@ -97,7 +97,7 @@ class DouyinDbStoreImplement(AbstractStore):
|
||||
Args:
|
||||
content_item: content item dict
|
||||
"""
|
||||
aweme_id = int(content_item.get("aweme_id"))
|
||||
aweme_id = content_item.get("aweme_id")
|
||||
async with get_session() as session:
|
||||
result = await session.execute(select(DouyinAweme).where(DouyinAweme.aweme_id == aweme_id))
|
||||
aweme_detail = result.scalar_one_or_none()
|
||||
@@ -118,7 +118,7 @@ class DouyinDbStoreImplement(AbstractStore):
|
||||
Args:
|
||||
comment_item: comment item dict
|
||||
"""
|
||||
comment_id = int(comment_item.get("comment_id"))
|
||||
comment_id = comment_item.get("comment_id")
|
||||
async with get_session() as session:
|
||||
result = await session.execute(select(DouyinAwemeComment).where(DouyinAwemeComment.comment_id == comment_id))
|
||||
comment_detail = result.scalar_one_or_none()
|
||||
|
||||
@@ -118,8 +118,7 @@ class WeiboDbStoreImplement(AbstractStore):
|
||||
"""
|
||||
# 教学版兜底:过滤掉已删除/多余字段,确保不会把 user_id/avatar 等传给 ORM
|
||||
content_item = _filter_model_fields(WeiboNote, content_item)
|
||||
note_id = int(content_item.get("note_id"))
|
||||
content_item["note_id"] = note_id
|
||||
note_id = content_item.get("note_id")
|
||||
async with get_session() as session:
|
||||
stmt = select(WeiboNote).where(WeiboNote.note_id == note_id)
|
||||
res = await session.execute(stmt)
|
||||
@@ -147,9 +146,7 @@ class WeiboDbStoreImplement(AbstractStore):
|
||||
"""
|
||||
# 教学版兜底:过滤掉已删除/多余字段,确保不会把 user_id/avatar 等传给 ORM
|
||||
comment_item = _filter_model_fields(WeiboNoteComment, comment_item)
|
||||
comment_id = int(comment_item.get("comment_id"))
|
||||
comment_item["comment_id"] = comment_id
|
||||
comment_item["note_id"] = int(comment_item.get("note_id", 0) or 0)
|
||||
comment_id = comment_item.get("comment_id")
|
||||
comment_item["create_time"] = int(comment_item.get("create_time", 0) or 0)
|
||||
comment_item["comment_like_count"] = str(comment_item.get("comment_like_count", "0"))
|
||||
comment_item["sub_comment_count"] = str(comment_item.get("sub_comment_count", "0"))
|
||||
|
||||
@@ -170,7 +170,7 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
|
||||
"nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏)
|
||||
"sub_comment_count": comment_item.get("sub_comment_count", 0), # Sub-comment count
|
||||
"pictures": ",".join(comment_pictures), # Comment pictures
|
||||
"parent_comment_id": target_comment.get("id", 0), # Parent comment ID
|
||||
"parent_comment_id": target_comment.get("id", ""), # Parent comment ID
|
||||
"last_modify_ts": utils.get_current_timestamp(), # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage)
|
||||
"like_count": comment_item.get("like_count", 0),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user