fix: 将平台业务ID统一改为String并自动建表

- 抖音、B站、快手、微博的帖子/视频/评论ID从BigInteger改为String,
  避免PostgreSQL下字符串写入BIGINT报错及未来ID溢出风险
- B站dynamic_id改为String,修复API返回id_str被强转int导致的精度丢失
- 知乎提取器对content_id/question_id显式str()转换
- main.py启动数据库保存模式时自动建表,无需手动--init_db
- 同步更新相关老化测试
This commit is contained in:
程序员阿江(Relakkes)
2026-07-01 23:02:13 +08:00
parent 51d4853db5
commit a06273ea6c
11 changed files with 55 additions and 61 deletions

View File

@@ -282,7 +282,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
# dict 多了已删列会 TypeError,少了非空必填列 SQLAlchemy 也会报错;
# 此处证明 captured 的 key 与删列后 ORM 列完全对得上,不抛异常。
obj = DouyinAweme(**captured)
assert int(obj.aweme_id) == int(aweme["aweme_id"])
assert obj.aweme_id == aweme["aweme_id"]
assert obj.creator_hash == anonymize_user_id(raw_uid)
assert obj.creator_hash != raw_uid
assert obj.nickname == mask_nickname(raw_nick)
@@ -310,7 +310,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
# 查询回读
async with db_session.get_session() as session:
res = await session.execute(
select(DouyinAweme).where(DouyinAweme.aweme_id == int(aweme["aweme_id"]))
select(DouyinAweme).where(DouyinAweme.aweme_id == aweme["aweme_id"])
)
row = res.scalar_one_or_none()
await engine.dispose()
@@ -320,7 +320,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
# ---- 4. 回读断言 ----
assert row is not None, "作品未写入 SQLite"
assert int(row.aweme_id) == int(aweme["aweme_id"])
assert row.aweme_id == aweme["aweme_id"]
assert row.creator_hash == anonymize_user_id(raw_uid)
assert row.creator_hash != raw_uid
assert row.nickname == mask_nickname(raw_nick)
@@ -348,7 +348,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
ds.DouyinStoreFactory.create_store = orig_c
captured_comment = fake_c.comments[0]
comment_obj = DouyinAwemeComment(**captured_comment) # 不抛异常即对得上
assert int(comment_obj.comment_id) == int(comment["cid"])
assert comment_obj.comment_id == comment["cid"]
assert comment_obj.creator_hash == anonymize_user_id(comment["user"]["uid"])
assert comment_obj.nickname == mask_nickname(comment["user"]["nickname"])
_assert_no_forbidden(captured_comment, "douyin_comment_orm_construct")

View File

@@ -33,6 +33,7 @@ from store.xhs._store_impl import (
XhsMongoStoreImplement,
XhsExcelStoreImplement
)
from store.excel_store_base import ExcelStoreBase
class TestXhsStoreFactory:
@@ -73,7 +74,8 @@ class TestXhsStoreFactory:
"""Test creating Excel store"""
# ContextVar cannot be mocked, so we test with actual value
store = XhsStoreFactory.create_store()
assert isinstance(store, XhsExcelStoreImplement)
# XhsExcelStoreImplement 返回 ExcelStoreBase 单例,实际类型为 ExcelStoreBase
assert isinstance(store, ExcelStoreBase)
@patch('config.SAVE_DATA_OPTION', 'jsonl')
def test_create_jsonl_store(self):

View File

@@ -20,7 +20,7 @@ def test_extract_search_note_list_from_keyword_page():
assert notes[0].note_id == "9117888152"
assert notes[0].title.startswith("武汉交互空间科技")
assert notes[0].tieba_name == "武汉交互空间"
assert notes[0].user_nickname == "VR虚拟达"
assert notes[0].user_nickname == "V***"
def test_extract_search_note_list_from_current_pc_card_page():
@@ -56,7 +56,7 @@ def test_extract_search_note_list_from_current_pc_card_page():
assert notes[0].desc == "培训班需求,数学,英语,编程老师,专职兼职都可"
assert notes[0].tieba_name == "诸城吧"
assert notes[0].tieba_link.endswith("kw=%E8%AF%B8%E5%9F%8E")
assert notes[0].user_nickname == "754023117"
assert notes[0].user_nickname == "7***7"
assert notes[0].publish_time == "2026-3-15"
assert notes[0].total_replay_num == 19
@@ -147,17 +147,17 @@ def test_extract_note_detail_and_comments_from_current_pc_api():
assert note.note_id == "10451142633"
assert note.title == "这X尔斯对比巴尔斯我只能说ID正确允许居功自傲"
assert note.desc == "皮队败决处刑德国编程钢琴师兼职数学家"
assert note.user_nickname == "高祖蒙斯"
assert note.user_nickname == "***"
assert note.tieba_name == "dota2吧"
assert note.total_replay_num == 15
assert note.total_replay_page == 1
assert note.ip_location == "广东"
# 教学版已移除 ip_location 等可定位真人字段
assert len(comments) == 1
assert comments[0].comment_id == "153154097267"
assert comments[0].content == "xg现在大树阵容另一个辅助不选控制"
assert comments[0].user_nickname == "胡希3"
assert comments[0].user_nickname == "***3"
assert comments[0].sub_comment_count == 4
assert comments[0].ip_location == "河北"
# 教学版已移除 ip_location 等可定位真人字段
def test_extract_creator_info_and_threads_from_current_pc_api():
@@ -191,12 +191,10 @@ def test_extract_creator_info_and_threads_from_current_pc_api():
creator = extractor.extract_creator_info_from_api(creator_api)
thread_ids = extractor.extract_creator_thread_id_list_from_api(feed_api)
assert creator.user_id == "3546493137"
assert creator.user_name == "拜月教Alice"
assert creator.nickname == "米米世界大手子"
assert creator.user_nickname == "米***子"
assert creator.fans == 58
assert creator.follows == 1
assert creator.ip_location == "广东"
# 教学版已移除 user_id、user_name、ip_location 等可定位真人字段
assert creator.registration_duration == "7.8"
assert thread_ids == ["10208192951", "9835114923"]
@@ -225,7 +223,7 @@ def test_extract_tieba_note_list_from_bigpipe_thread_page():
assert len(notes) == 48
assert notes[0].note_id == "9079949995"
assert notes[0].title == "盗墓笔记全集+txt小说已整理"
assert notes[0].user_nickname == "子伯"
assert notes[0].user_nickname == "***"
assert notes[0].tieba_name == "盗墓笔记吧"
assert notes[0].tieba_link.endswith("kw=%E7%9B%97%E5%A2%93%E7%AC%94%E8%AE%B0&ie=utf-8")
@@ -235,11 +233,11 @@ def test_extract_note_detail_from_post_page():
assert note.note_id == "9117905169"
assert note.title == "对于一个父亲来说这个女儿14岁就死了"
assert note.user_nickname == ""
assert note.user_nickname == "***"
assert note.tieba_name == "以太比特吧"
assert note.total_replay_num == 786
assert note.total_replay_page == 13
assert note.ip_location == "广东"
# 教学版已移除 ip_location 等可定位真人字段
def test_extract_parent_comments_from_post_page():
@@ -251,9 +249,9 @@ def test_extract_parent_comments_from_post_page():
assert len(comments) == 30
assert comments[0].comment_id == "150726491368"
assert comments[0].content == "中国队第22金无悬念"
assert comments[0].user_nickname == "heinzfrentzen"
assert comments[0].user_nickname == "h***n"
assert comments[0].tieba_name == "网球风云吧"
assert comments[0].ip_location == "福建"
# 教学版已移除 ip_location 等可定位真人字段
def test_extract_sub_comments_with_class_token_matching():
@@ -275,4 +273,4 @@ def test_extract_sub_comments_with_class_token_matching():
assert len(comments) >= 10
assert comments[0].comment_id
assert comments[0].parent_comment_id == parent.comment_id
assert comments[0].user_link.startswith("https://tieba.baidu.com/home/main")
# 教学版已移除 user_link 等可定位真人字段的采集

View File

@@ -27,8 +27,8 @@ RAW_USER_ID = 7654321
RAW_NICKNAME = "微博达人"
RAW_COMMENT_USER_ID = 111222
RAW_COMMENT_NICKNAME = "评论员小张"
NOTE_ID = 5123456789
COMMENT_ID = 998877
NOTE_ID = "5123456789"
COMMENT_ID = "998877"
# 合法 RFC2822 时间串(weekday 与日期已对齐:2025-06-14 是周六)
RFC2822_TIME = "Sat Jun 14 12:00:00 +0800 2025"
@@ -168,7 +168,7 @@ def test_weibo_comment_masks_user_info():
fake = _FakeStore()
wb_, orig = _patch_factory(fake)
try:
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
asyncio.run(wb.update_weibo_note_comment(NOTE_ID, make_mock_comment()))
finally:
_restore(wb_, orig)
@@ -194,7 +194,7 @@ def test_weibo_comment_masks_user_info():
# 4. 内容字段正确
assert "说得好" in captured["content"]
assert captured["comment_id"] == str(COMMENT_ID)
assert captured["comment_id"] == COMMENT_ID
assert captured["comment_like_count"] == "5"
assert captured["sub_comment_count"] == "3"
assert captured["parent_comment_id"] == "parent_abc"
@@ -218,7 +218,7 @@ def test_weibo_store_end_to_end_sqlite():
wb_, orig = _patch_factory(fake)
try:
asyncio.run(wb.update_weibo_note(make_mock_note()))
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
asyncio.run(wb.update_weibo_note_comment(NOTE_ID, make_mock_comment()))
finally:
_restore(wb_, orig)
@@ -250,11 +250,8 @@ def test_weibo_store_end_to_end_sqlite():
# 确认没有 desc 列存任何用户描述
assert "desc" not in note_cols
# ---- 4. comment:同上。注意 store_comment 实现会把 comment_id/note_id/create_time
# 转成 int 后再构造 ORM,此处忠实复刻该转换以走通 BigInteger 列写入 ----
# ---- 4. comment:同上。ID 已统一为字符串类型,create_time 保持 int ----
cc = dict(captured_comment)
cc["comment_id"] = int(cc["comment_id"])
cc["note_id"] = int(cc.get("note_id", 0) or 0)
cc["create_time"] = int(cc.get("create_time", 0) or 0)
comment_obj = WeiboNoteComment(**cc) # 不抛异常 => key 与 ORM 列对得上
session.add(comment_obj)