fix: 将平台业务ID统一改为String并自动建表

- 抖音、B站、快手、微博的帖子/视频/评论ID从BigInteger改为String，
  避免PostgreSQL下字符串写入BIGINT报错及未来ID溢出风险
- B站dynamic_id改为String，修复API返回id_str被强转int导致的精度丢失
- 知乎提取器对content_id/question_id显式str()转换
- main.py启动数据库保存模式时自动建表，无需手动--init_db
- 同步更新相关老化测试

database/models.py

@@ -33,7 +33,7 @@ Base = declarative_base()
 class BilibiliVideo(Base):
     __tablename__ = 'bilibili_video'
     id = Column(Integer, primary_key=True, comment='主键ID')
-    video_id = Column(BigInteger, nullable=False, index=True, unique=True, comment='视频ID')
+    video_id = Column(String(64), nullable=False, index=True, unique=True, comment='视频ID')
     video_url = Column(Text, nullable=False, comment='视频URL')
     creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
     nickname = Column(Text, comment='用户昵称(已脱敏)')
@@ -61,8 +61,8 @@ class BilibiliVideoComment(Base):
     nickname = Column(Text, comment='用户昵称(已脱敏)')
     add_ts = Column(BigInteger, comment='添加时间戳')
     last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
-    comment_id = Column(BigInteger, index=True, comment='评论ID')
+    comment_id = Column(String(128), index=True, comment='评论ID')
-    video_id = Column(BigInteger, index=True, comment='视频ID')
+    video_id = Column(String(64), index=True, comment='视频ID')
     content = Column(Text, comment='评论内容')
     create_time = Column(BigInteger, comment='创建时间戳')
     sub_comment_count = Column(Text, comment='子评论数')
@@ -72,7 +72,7 @@ class BilibiliVideoComment(Base):
 class BilibiliUpDynamic(Base):
     __tablename__ = 'bilibili_up_dynamic'
     id = Column(Integer, primary_key=True, comment='主键ID')
-    dynamic_id = Column(BigInteger, index=True, comment='动态ID')
+    dynamic_id = Column(String(128), index=True, comment='动态ID')
     creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
     user_name = Column(Text, comment='用户名称(已脱敏)')
     text = Column(Text, comment='动态内容')
@@ -91,7 +91,7 @@ class DouyinAweme(Base):
     nickname = Column(Text, comment='用户昵称(已脱敏)')
     add_ts = Column(BigInteger, comment='添加时间戳')
     last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
-    aweme_id = Column(BigInteger, index=True, comment='作品ID')
+    aweme_id = Column(String(255), index=True, comment='作品ID')
     aweme_type = Column(Text, comment='作品类型')
     title = Column(Text, comment='作品标题')
     desc = Column(Text, comment='作品描述')
@@ -114,8 +114,8 @@ class DouyinAwemeComment(Base):
     nickname = Column(Text, comment='用户昵称(已脱敏)')
     add_ts = Column(BigInteger, comment='添加时间戳')
     last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
-    comment_id = Column(BigInteger, index=True, comment='评论ID')
+    comment_id = Column(String(255), index=True, comment='评论ID')
-    aweme_id = Column(BigInteger, index=True, comment='作品ID')
+    aweme_id = Column(String(255), index=True, comment='作品ID')
     content = Column(Text, comment='评论内容')
     create_time = Column(BigInteger, comment='创建时间戳')
     sub_comment_count = Column(Text, comment='子评论数')
@@ -149,7 +149,7 @@ class KuaishouVideoComment(Base):
     nickname = Column(Text, comment='用户昵称(已脱敏)')
     add_ts = Column(BigInteger, comment='添加时间戳')
     last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
-    comment_id = Column(BigInteger, index=True, comment='评论ID')
+    comment_id = Column(String(255), index=True, comment='评论ID')
     video_id = Column(String(255), index=True, comment='视频ID')
     content = Column(Text, comment='评论内容')
     create_time = Column(BigInteger, comment='创建时间戳')
@@ -162,7 +162,7 @@ class WeiboNote(Base):
     nickname = Column(Text, comment='用户昵称(已脱敏)')
     add_ts = Column(BigInteger, comment='添加时间戳')
     last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
-    note_id = Column(BigInteger, index=True, comment='笔记ID')
+    note_id = Column(String(64), index=True, comment='笔记ID')
     content = Column(Text, comment='笔记内容')
     create_time = Column(BigInteger, index=True, comment='创建时间戳')
     create_date_time = Column(String(255), index=True, comment='创建日期时间')
@@ -179,8 +179,8 @@ class WeiboNoteComment(Base):
     nickname = Column(Text, comment='用户昵称(已脱敏)')
     add_ts = Column(BigInteger, comment='添加时间戳')
     last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
-    comment_id = Column(BigInteger, index=True, comment='评论ID')
+    comment_id = Column(String(64), index=True, comment='评论ID')
-    note_id = Column(BigInteger, index=True, comment='笔记ID')
+    note_id = Column(String(64), index=True, comment='笔记ID')
     content = Column(Text, comment='评论内容')
     create_time = Column(BigInteger, comment='创建时间戳')
     create_date_time = Column(String(255), index=True, comment='创建日期时间')

main.py

@@ -106,6 +106,10 @@ async def main() -> None:
         print(f"Database {args.init_db} initialized successfully.")
         return
 
+    # 数据库保存模式下自动建表，避免首次运行时出现 no such table 错误
+    if config.SAVE_DATA_OPTION in ("sqlite", "mysql", "db", "postgres"):
+        await db.init_db(config.SAVE_DATA_OPTION)
+
     crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM)
     await crawler.start()
 

media_platform/zhihu/help.py

@@ -107,10 +107,10 @@ class ZhihuExtractor:
         Returns:
         """
         res = ZhihuContent()
-        res.content_id = answer.get("id")
+        res.content_id = str(answer.get("id") or "")
         res.content_type = answer.get("type")
         res.content_text = extract_text_from_html(answer.get("content", ""))
-        res.question_id = answer.get("question").get("id")
+        res.question_id = str(answer.get("question", {}).get("id") or "")
         res.content_url = f"{zhihu_constant.ZHIHU_URL}/question/{res.question_id}/answer/{res.content_id}"
         res.title = extract_text_from_html(answer.get("title", ""))
         res.desc = extract_text_from_html(answer.get("description", "") or answer.get("excerpt", ""))
@@ -135,7 +135,7 @@ class ZhihuExtractor:
 
         """
         res = ZhihuContent()
-        res.content_id = article.get("id")
+        res.content_id = str(article.get("id") or "")
         res.content_type = article.get("type")
         res.content_text = extract_text_from_html(article.get("content"))
         res.content_url = f"{zhihu_constant.ZHIHU_ZHUANLAN_URL}/p/{res.content_id}"
@@ -162,6 +162,8 @@ class ZhihuExtractor:
 
         """
         res = ZhihuContent()
+        res.content_id = str(zvideo.get("id") or "")
+        res.content_type = zvideo.get("type")
 
         if "video" in zvideo and isinstance(zvideo.get("video"), dict): # This indicates data from the creator's homepage video list API
             res.content_url = f"{zhihu_constant.ZHIHU_URL}/zvideo/{res.content_id}"
@@ -170,8 +172,6 @@ class ZhihuExtractor:
         else:
             res.content_url = zvideo.get("video_url")
             res.created_time = zvideo.get("created_at")
-        res.content_id = zvideo.get("id")
-        res.content_type = zvideo.get("type")
         res.title = extract_text_from_html(zvideo.get("title"))
         res.desc = extract_text_from_html(zvideo.get("description"))
         res.voteup_count = zvideo.get("voteup_count")

store/bilibili/_store_impl.py

@@ -128,8 +128,7 @@ class BiliDbStoreImplement(AbstractStore):
         Args:
             content_item: content item dict
         """
-        video_id = int(content_item.get("video_id"))
+        video_id = content_item.get("video_id")
-        content_item["video_id"] = video_id
         content_item["liked_count"] = int(content_item.get("liked_count", 0) or 0)
         content_item["create_time"] = int(content_item.get("create_time", 0) or 0)
 
@@ -154,9 +153,7 @@ class BiliDbStoreImplement(AbstractStore):
         Args:
             comment_item: comment item dict
         """
-        comment_id = int(comment_item.get("comment_id"))
+        comment_id = comment_item.get("comment_id")
-        comment_item["comment_id"] = comment_id
-        comment_item["video_id"] = int(comment_item.get("video_id", 0) or 0)
         comment_item["create_time"] = int(comment_item.get("create_time", 0) or 0)
         comment_item["like_count"] = str(comment_item.get("like_count", "0"))
         comment_item["sub_comment_count"] = str(comment_item.get("sub_comment_count", "0"))
@@ -191,8 +188,7 @@ class BiliDbStoreImplement(AbstractStore):
         Args:
             dynamic_item: dynamic item dict
         """
-        dynamic_id = int(dynamic_item.get("dynamic_id"))
+        dynamic_id = dynamic_item.get("dynamic_id")
-        dynamic_item["dynamic_id"] = dynamic_id
 
         async with get_session() as session:
             result = await session.execute(select(BilibiliUpDynamic).where(BilibiliUpDynamic.dynamic_id == dynamic_id))

store/douyin/_store_impl.py

@@ -97,7 +97,7 @@ class DouyinDbStoreImplement(AbstractStore):
         Args:
             content_item: content item dict
         """
-        aweme_id = int(content_item.get("aweme_id"))
+        aweme_id = content_item.get("aweme_id")
         async with get_session() as session:
             result = await session.execute(select(DouyinAweme).where(DouyinAweme.aweme_id == aweme_id))
             aweme_detail = result.scalar_one_or_none()
@@ -118,7 +118,7 @@ class DouyinDbStoreImplement(AbstractStore):
         Args:
             comment_item: comment item dict
         """
-        comment_id = int(comment_item.get("comment_id"))
+        comment_id = comment_item.get("comment_id")
         async with get_session() as session:
             result = await session.execute(select(DouyinAwemeComment).where(DouyinAwemeComment.comment_id == comment_id))
             comment_detail = result.scalar_one_or_none()

store/weibo/_store_impl.py

@@ -118,8 +118,7 @@ class WeiboDbStoreImplement(AbstractStore):
         """
         # 教学版兜底：过滤掉已删除/多余字段，确保不会把 user_id/avatar 等传给 ORM
         content_item = _filter_model_fields(WeiboNote, content_item)
-        note_id = int(content_item.get("note_id"))
+        note_id = content_item.get("note_id")
-        content_item["note_id"] = note_id
         async with get_session() as session:
             stmt = select(WeiboNote).where(WeiboNote.note_id == note_id)
             res = await session.execute(stmt)
@@ -147,9 +146,7 @@ class WeiboDbStoreImplement(AbstractStore):
         """
         # 教学版兜底：过滤掉已删除/多余字段，确保不会把 user_id/avatar 等传给 ORM
         comment_item = _filter_model_fields(WeiboNoteComment, comment_item)
-        comment_id = int(comment_item.get("comment_id"))
+        comment_id = comment_item.get("comment_id")
-        comment_item["comment_id"] = comment_id
-        comment_item["note_id"] = int(comment_item.get("note_id", 0) or 0)
         comment_item["create_time"] = int(comment_item.get("create_time", 0) or 0)
         comment_item["comment_like_count"] = str(comment_item.get("comment_like_count", "0"))
         comment_item["sub_comment_count"] = str(comment_item.get("sub_comment_count", "0"))

store/xhs/__init__.py

@@ -170,7 +170,7 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
         "nickname": mask_nickname(user_info.get("nickname")),  # 用户昵称(已脱敏)
         "sub_comment_count": comment_item.get("sub_comment_count", 0),  # Sub-comment count
         "pictures": ",".join(comment_pictures),  # Comment pictures
-        "parent_comment_id": target_comment.get("id", 0),  # Parent comment ID
+        "parent_comment_id": target_comment.get("id", ""),  # Parent comment ID
         "last_modify_ts": utils.get_current_timestamp(),  # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage)
         "like_count": comment_item.get("like_count", 0),
     }

tests/test_douyin_no_user_info.py

@@ -282,7 +282,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
     # dict 多了已删列会 TypeError,少了非空必填列 SQLAlchemy 也会报错;
     # 此处证明 captured 的 key 与删列后 ORM 列完全对得上,不抛异常。
     obj = DouyinAweme(**captured)
-    assert int(obj.aweme_id) == int(aweme["aweme_id"])
+    assert obj.aweme_id == aweme["aweme_id"]
     assert obj.creator_hash == anonymize_user_id(raw_uid)
     assert obj.creator_hash != raw_uid
     assert obj.nickname == mask_nickname(raw_nick)
@@ -310,7 +310,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
         # 查询回读
         async with db_session.get_session() as session:
             res = await session.execute(
-                select(DouyinAweme).where(DouyinAweme.aweme_id == int(aweme["aweme_id"]))
+                select(DouyinAweme).where(DouyinAweme.aweme_id == aweme["aweme_id"])
             )
             row = res.scalar_one_or_none()
         await engine.dispose()
@@ -320,7 +320,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
 
     # ---- 4. 回读断言 ----
     assert row is not None, "作品未写入 SQLite"
-    assert int(row.aweme_id) == int(aweme["aweme_id"])
+    assert row.aweme_id == aweme["aweme_id"]
     assert row.creator_hash == anonymize_user_id(raw_uid)
     assert row.creator_hash != raw_uid
     assert row.nickname == mask_nickname(raw_nick)
@@ -348,7 +348,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
         ds.DouyinStoreFactory.create_store = orig_c
     captured_comment = fake_c.comments[0]
     comment_obj = DouyinAwemeComment(**captured_comment)  # 不抛异常即对得上
-    assert int(comment_obj.comment_id) == int(comment["cid"])
+    assert comment_obj.comment_id == comment["cid"]
     assert comment_obj.creator_hash == anonymize_user_id(comment["user"]["uid"])
     assert comment_obj.nickname == mask_nickname(comment["user"]["nickname"])
     _assert_no_forbidden(captured_comment, "douyin_comment_orm_construct")

tests/test_store_factory.py

@@ -33,6 +33,7 @@ from store.xhs._store_impl import (
     XhsMongoStoreImplement,
     XhsExcelStoreImplement
 )
+from store.excel_store_base import ExcelStoreBase
 
 
 class TestXhsStoreFactory:
@@ -73,7 +74,8 @@ class TestXhsStoreFactory:
         """Test creating Excel store"""
         # ContextVar cannot be mocked, so we test with actual value
         store = XhsStoreFactory.create_store()
-        assert isinstance(store, XhsExcelStoreImplement)
+        # XhsExcelStoreImplement 返回 ExcelStoreBase 单例，实际类型为 ExcelStoreBase
+        assert isinstance(store, ExcelStoreBase)
 
     @patch('config.SAVE_DATA_OPTION', 'jsonl')
     def test_create_jsonl_store(self):

tests/test_tieba_extractor.py

@@ -20,7 +20,7 @@ def test_extract_search_note_list_from_keyword_page():
     assert notes[0].note_id == "9117888152"
     assert notes[0].title.startswith("武汉交互空间科技")
     assert notes[0].tieba_name == "武汉交互空间"
-    assert notes[0].user_nickname == "VR虚拟达人"
+    assert notes[0].user_nickname == "V***人"
 
 
 def test_extract_search_note_list_from_current_pc_card_page():
@@ -56,7 +56,7 @@ def test_extract_search_note_list_from_current_pc_card_page():
     assert notes[0].desc == "培训班需求，数学，英语，编程老师，专职兼职都可"
     assert notes[0].tieba_name == "诸城吧"
     assert notes[0].tieba_link.endswith("kw=%E8%AF%B8%E5%9F%8E")
-    assert notes[0].user_nickname == "754023117"
+    assert notes[0].user_nickname == "7***7"
     assert notes[0].publish_time == "2026-3-15"
     assert notes[0].total_replay_num == 19
 
@@ -147,17 +147,17 @@ def test_extract_note_detail_and_comments_from_current_pc_api():
     assert note.note_id == "10451142633"
     assert note.title == "这X尔斯对比巴尔斯，我只能说ID正确，允许居功自傲"
     assert note.desc == "皮队败决处刑德国编程钢琴师兼职数学家"
-    assert note.user_nickname == "泰高祖蒙斯克"
+    assert note.user_nickname == "泰***克"
     assert note.tieba_name == "dota2吧"
     assert note.total_replay_num == 15
     assert note.total_replay_page == 1
-    assert note.ip_location == "广东"
+    # 教学版已移除 ip_location 等可定位真人字段
     assert len(comments) == 1
     assert comments[0].comment_id == "153154097267"
     assert comments[0].content == "xg现在大树阵容另一个辅助不选控制"
-    assert comments[0].user_nickname == "期胡希3"
+    assert comments[0].user_nickname == "期***3"
     assert comments[0].sub_comment_count == 4
-    assert comments[0].ip_location == "河北"
+    # 教学版已移除 ip_location 等可定位真人字段
 
 
 def test_extract_creator_info_and_threads_from_current_pc_api():
@@ -191,12 +191,10 @@ def test_extract_creator_info_and_threads_from_current_pc_api():
     creator = extractor.extract_creator_info_from_api(creator_api)
     thread_ids = extractor.extract_creator_thread_id_list_from_api(feed_api)
 
-    assert creator.user_id == "3546493137"
+    assert creator.user_nickname == "米***子"
-    assert creator.user_name == "拜月教Alice"
-    assert creator.nickname == "米米世界大手子"
     assert creator.fans == 58
     assert creator.follows == 1
-    assert creator.ip_location == "广东"
+    # 教学版已移除 user_id、user_name、ip_location 等可定位真人字段
     assert creator.registration_duration == "7.8"
     assert thread_ids == ["10208192951", "9835114923"]
 
@@ -225,7 +223,7 @@ def test_extract_tieba_note_list_from_bigpipe_thread_page():
     assert len(notes) == 48
     assert notes[0].note_id == "9079949995"
     assert notes[0].title == "盗墓笔记全集+txt小说，已整理"
-    assert notes[0].user_nickname == "公子伯仲"
+    assert notes[0].user_nickname == "公***仲"
     assert notes[0].tieba_name == "盗墓笔记吧"
     assert notes[0].tieba_link.endswith("kw=%E7%9B%97%E5%A2%93%E7%AC%94%E8%AE%B0&ie=utf-8")
 
@@ -235,11 +233,11 @@ def test_extract_note_detail_from_post_page():
 
     assert note.note_id == "9117905169"
     assert note.title == "对于一个父亲来说，这个女儿14岁就死了"
-    assert note.user_nickname == "章景轩"
+    assert note.user_nickname == "章***轩"
     assert note.tieba_name == "以太比特吧"
     assert note.total_replay_num == 786
     assert note.total_replay_page == 13
-    assert note.ip_location == "广东"
+    # 教学版已移除 ip_location 等可定位真人字段
 
 
 def test_extract_parent_comments_from_post_page():
@@ -251,9 +249,9 @@ def test_extract_parent_comments_from_post_page():
     assert len(comments) == 30
     assert comments[0].comment_id == "150726491368"
     assert comments[0].content == "中国队第22金！无悬念！"
-    assert comments[0].user_nickname == "heinzfrentzen"
+    assert comments[0].user_nickname == "h***n"
     assert comments[0].tieba_name == "网球风云吧"
-    assert comments[0].ip_location == "福建"
+    # 教学版已移除 ip_location 等可定位真人字段
 
 
 def test_extract_sub_comments_with_class_token_matching():
@@ -275,4 +273,4 @@ def test_extract_sub_comments_with_class_token_matching():
     assert len(comments) >= 10
     assert comments[0].comment_id
     assert comments[0].parent_comment_id == parent.comment_id
-    assert comments[0].user_link.startswith("https://tieba.baidu.com/home/main")
+    # 教学版已移除 user_link 等可定位真人字段的采集

tests/test_weibo_no_user_info.py

@@ -27,8 +27,8 @@ RAW_USER_ID = 7654321
 RAW_NICKNAME = "微博达人"
 RAW_COMMENT_USER_ID = 111222
 RAW_COMMENT_NICKNAME = "评论员小张"
-NOTE_ID = 5123456789
+NOTE_ID = "5123456789"
-COMMENT_ID = 998877
+COMMENT_ID = "998877"
 # 合法 RFC2822 时间串(weekday 与日期已对齐:2025-06-14 是周六)
 RFC2822_TIME = "Sat Jun 14 12:00:00 +0800 2025"
 
@@ -168,7 +168,7 @@ def test_weibo_comment_masks_user_info():
     fake = _FakeStore()
     wb_, orig = _patch_factory(fake)
     try:
-        asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
+        asyncio.run(wb.update_weibo_note_comment(NOTE_ID, make_mock_comment()))
     finally:
         _restore(wb_, orig)
 
@@ -194,7 +194,7 @@ def test_weibo_comment_masks_user_info():
 
     # 4. 内容字段正确
     assert "说得好" in captured["content"]
-    assert captured["comment_id"] == str(COMMENT_ID)
+    assert captured["comment_id"] == COMMENT_ID
     assert captured["comment_like_count"] == "5"
     assert captured["sub_comment_count"] == "3"
     assert captured["parent_comment_id"] == "parent_abc"
@@ -218,7 +218,7 @@ def test_weibo_store_end_to_end_sqlite():
     wb_, orig = _patch_factory(fake)
     try:
         asyncio.run(wb.update_weibo_note(make_mock_note()))
-        asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
+        asyncio.run(wb.update_weibo_note_comment(NOTE_ID, make_mock_comment()))
     finally:
         _restore(wb_, orig)
 
@@ -250,11 +250,8 @@ def test_weibo_store_end_to_end_sqlite():
         # 确认没有 desc 列存任何用户描述
         assert "desc" not in note_cols
 
-        # ---- 4. comment:同上。注意 store_comment 实现会把 comment_id/note_id/create_time
+        # ---- 4. comment:同上。ID 已统一为字符串类型,create_time 保持 int ----
-        #         转成 int 后再构造 ORM,此处忠实复刻该转换以走通 BigInteger 列写入 ----
         cc = dict(captured_comment)
-        cc["comment_id"] = int(cc["comment_id"])
-        cc["note_id"] = int(cc.get("note_id", 0) or 0)
         cc["create_time"] = int(cc.get("create_time", 0) or 0)
         comment_obj = WeiboNoteComment(**cc)  # 不抛异常 => key 与 ORM 列对得上
         session.add(comment_obj)