mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-07-28 07:20:25 +08:00
fix: 将平台业务ID统一改为String并自动建表
- 抖音、B站、快手、微博的帖子/视频/评论ID从BigInteger改为String, 避免PostgreSQL下字符串写入BIGINT报错及未来ID溢出风险 - B站dynamic_id改为String,修复API返回id_str被强转int导致的精度丢失 - 知乎提取器对content_id/question_id显式str()转换 - main.py启动数据库保存模式时自动建表,无需手动--init_db - 同步更新相关老化测试
This commit is contained in:
@@ -282,7 +282,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
|
||||
# dict 多了已删列会 TypeError,少了非空必填列 SQLAlchemy 也会报错;
|
||||
# 此处证明 captured 的 key 与删列后 ORM 列完全对得上,不抛异常。
|
||||
obj = DouyinAweme(**captured)
|
||||
assert int(obj.aweme_id) == int(aweme["aweme_id"])
|
||||
assert obj.aweme_id == aweme["aweme_id"]
|
||||
assert obj.creator_hash == anonymize_user_id(raw_uid)
|
||||
assert obj.creator_hash != raw_uid
|
||||
assert obj.nickname == mask_nickname(raw_nick)
|
||||
@@ -310,7 +310,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
|
||||
# 查询回读
|
||||
async with db_session.get_session() as session:
|
||||
res = await session.execute(
|
||||
select(DouyinAweme).where(DouyinAweme.aweme_id == int(aweme["aweme_id"]))
|
||||
select(DouyinAweme).where(DouyinAweme.aweme_id == aweme["aweme_id"])
|
||||
)
|
||||
row = res.scalar_one_or_none()
|
||||
await engine.dispose()
|
||||
@@ -320,7 +320,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
|
||||
|
||||
# ---- 4. 回读断言 ----
|
||||
assert row is not None, "作品未写入 SQLite"
|
||||
assert int(row.aweme_id) == int(aweme["aweme_id"])
|
||||
assert row.aweme_id == aweme["aweme_id"]
|
||||
assert row.creator_hash == anonymize_user_id(raw_uid)
|
||||
assert row.creator_hash != raw_uid
|
||||
assert row.nickname == mask_nickname(raw_nick)
|
||||
@@ -348,7 +348,7 @@ def test_douyin_store_end_to_end_sqlite(monkeypatch):
|
||||
ds.DouyinStoreFactory.create_store = orig_c
|
||||
captured_comment = fake_c.comments[0]
|
||||
comment_obj = DouyinAwemeComment(**captured_comment) # 不抛异常即对得上
|
||||
assert int(comment_obj.comment_id) == int(comment["cid"])
|
||||
assert comment_obj.comment_id == comment["cid"]
|
||||
assert comment_obj.creator_hash == anonymize_user_id(comment["user"]["uid"])
|
||||
assert comment_obj.nickname == mask_nickname(comment["user"]["nickname"])
|
||||
_assert_no_forbidden(captured_comment, "douyin_comment_orm_construct")
|
||||
|
||||
@@ -33,6 +33,7 @@ from store.xhs._store_impl import (
|
||||
XhsMongoStoreImplement,
|
||||
XhsExcelStoreImplement
|
||||
)
|
||||
from store.excel_store_base import ExcelStoreBase
|
||||
|
||||
|
||||
class TestXhsStoreFactory:
|
||||
@@ -73,7 +74,8 @@ class TestXhsStoreFactory:
|
||||
"""Test creating Excel store"""
|
||||
# ContextVar cannot be mocked, so we test with actual value
|
||||
store = XhsStoreFactory.create_store()
|
||||
assert isinstance(store, XhsExcelStoreImplement)
|
||||
# XhsExcelStoreImplement 返回 ExcelStoreBase 单例,实际类型为 ExcelStoreBase
|
||||
assert isinstance(store, ExcelStoreBase)
|
||||
|
||||
@patch('config.SAVE_DATA_OPTION', 'jsonl')
|
||||
def test_create_jsonl_store(self):
|
||||
|
||||
@@ -20,7 +20,7 @@ def test_extract_search_note_list_from_keyword_page():
|
||||
assert notes[0].note_id == "9117888152"
|
||||
assert notes[0].title.startswith("武汉交互空间科技")
|
||||
assert notes[0].tieba_name == "武汉交互空间"
|
||||
assert notes[0].user_nickname == "VR虚拟达人"
|
||||
assert notes[0].user_nickname == "V***人"
|
||||
|
||||
|
||||
def test_extract_search_note_list_from_current_pc_card_page():
|
||||
@@ -56,7 +56,7 @@ def test_extract_search_note_list_from_current_pc_card_page():
|
||||
assert notes[0].desc == "培训班需求,数学,英语,编程老师,专职兼职都可"
|
||||
assert notes[0].tieba_name == "诸城吧"
|
||||
assert notes[0].tieba_link.endswith("kw=%E8%AF%B8%E5%9F%8E")
|
||||
assert notes[0].user_nickname == "754023117"
|
||||
assert notes[0].user_nickname == "7***7"
|
||||
assert notes[0].publish_time == "2026-3-15"
|
||||
assert notes[0].total_replay_num == 19
|
||||
|
||||
@@ -147,17 +147,17 @@ def test_extract_note_detail_and_comments_from_current_pc_api():
|
||||
assert note.note_id == "10451142633"
|
||||
assert note.title == "这X尔斯对比巴尔斯,我只能说ID正确,允许居功自傲"
|
||||
assert note.desc == "皮队败决处刑德国编程钢琴师兼职数学家"
|
||||
assert note.user_nickname == "泰高祖蒙斯克"
|
||||
assert note.user_nickname == "泰***克"
|
||||
assert note.tieba_name == "dota2吧"
|
||||
assert note.total_replay_num == 15
|
||||
assert note.total_replay_page == 1
|
||||
assert note.ip_location == "广东"
|
||||
# 教学版已移除 ip_location 等可定位真人字段
|
||||
assert len(comments) == 1
|
||||
assert comments[0].comment_id == "153154097267"
|
||||
assert comments[0].content == "xg现在大树阵容另一个辅助不选控制"
|
||||
assert comments[0].user_nickname == "期胡希3"
|
||||
assert comments[0].user_nickname == "期***3"
|
||||
assert comments[0].sub_comment_count == 4
|
||||
assert comments[0].ip_location == "河北"
|
||||
# 教学版已移除 ip_location 等可定位真人字段
|
||||
|
||||
|
||||
def test_extract_creator_info_and_threads_from_current_pc_api():
|
||||
@@ -191,12 +191,10 @@ def test_extract_creator_info_and_threads_from_current_pc_api():
|
||||
creator = extractor.extract_creator_info_from_api(creator_api)
|
||||
thread_ids = extractor.extract_creator_thread_id_list_from_api(feed_api)
|
||||
|
||||
assert creator.user_id == "3546493137"
|
||||
assert creator.user_name == "拜月教Alice"
|
||||
assert creator.nickname == "米米世界大手子"
|
||||
assert creator.user_nickname == "米***子"
|
||||
assert creator.fans == 58
|
||||
assert creator.follows == 1
|
||||
assert creator.ip_location == "广东"
|
||||
# 教学版已移除 user_id、user_name、ip_location 等可定位真人字段
|
||||
assert creator.registration_duration == "7.8"
|
||||
assert thread_ids == ["10208192951", "9835114923"]
|
||||
|
||||
@@ -225,7 +223,7 @@ def test_extract_tieba_note_list_from_bigpipe_thread_page():
|
||||
assert len(notes) == 48
|
||||
assert notes[0].note_id == "9079949995"
|
||||
assert notes[0].title == "盗墓笔记全集+txt小说,已整理"
|
||||
assert notes[0].user_nickname == "公子伯仲"
|
||||
assert notes[0].user_nickname == "公***仲"
|
||||
assert notes[0].tieba_name == "盗墓笔记吧"
|
||||
assert notes[0].tieba_link.endswith("kw=%E7%9B%97%E5%A2%93%E7%AC%94%E8%AE%B0&ie=utf-8")
|
||||
|
||||
@@ -235,11 +233,11 @@ def test_extract_note_detail_from_post_page():
|
||||
|
||||
assert note.note_id == "9117905169"
|
||||
assert note.title == "对于一个父亲来说,这个女儿14岁就死了"
|
||||
assert note.user_nickname == "章景轩"
|
||||
assert note.user_nickname == "章***轩"
|
||||
assert note.tieba_name == "以太比特吧"
|
||||
assert note.total_replay_num == 786
|
||||
assert note.total_replay_page == 13
|
||||
assert note.ip_location == "广东"
|
||||
# 教学版已移除 ip_location 等可定位真人字段
|
||||
|
||||
|
||||
def test_extract_parent_comments_from_post_page():
|
||||
@@ -251,9 +249,9 @@ def test_extract_parent_comments_from_post_page():
|
||||
assert len(comments) == 30
|
||||
assert comments[0].comment_id == "150726491368"
|
||||
assert comments[0].content == "中国队第22金!无悬念!"
|
||||
assert comments[0].user_nickname == "heinzfrentzen"
|
||||
assert comments[0].user_nickname == "h***n"
|
||||
assert comments[0].tieba_name == "网球风云吧"
|
||||
assert comments[0].ip_location == "福建"
|
||||
# 教学版已移除 ip_location 等可定位真人字段
|
||||
|
||||
|
||||
def test_extract_sub_comments_with_class_token_matching():
|
||||
@@ -275,4 +273,4 @@ def test_extract_sub_comments_with_class_token_matching():
|
||||
assert len(comments) >= 10
|
||||
assert comments[0].comment_id
|
||||
assert comments[0].parent_comment_id == parent.comment_id
|
||||
assert comments[0].user_link.startswith("https://tieba.baidu.com/home/main")
|
||||
# 教学版已移除 user_link 等可定位真人字段的采集
|
||||
|
||||
@@ -27,8 +27,8 @@ RAW_USER_ID = 7654321
|
||||
RAW_NICKNAME = "微博达人"
|
||||
RAW_COMMENT_USER_ID = 111222
|
||||
RAW_COMMENT_NICKNAME = "评论员小张"
|
||||
NOTE_ID = 5123456789
|
||||
COMMENT_ID = 998877
|
||||
NOTE_ID = "5123456789"
|
||||
COMMENT_ID = "998877"
|
||||
# 合法 RFC2822 时间串(weekday 与日期已对齐:2025-06-14 是周六)
|
||||
RFC2822_TIME = "Sat Jun 14 12:00:00 +0800 2025"
|
||||
|
||||
@@ -168,7 +168,7 @@ def test_weibo_comment_masks_user_info():
|
||||
fake = _FakeStore()
|
||||
wb_, orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
|
||||
asyncio.run(wb.update_weibo_note_comment(NOTE_ID, make_mock_comment()))
|
||||
finally:
|
||||
_restore(wb_, orig)
|
||||
|
||||
@@ -194,7 +194,7 @@ def test_weibo_comment_masks_user_info():
|
||||
|
||||
# 4. 内容字段正确
|
||||
assert "说得好" in captured["content"]
|
||||
assert captured["comment_id"] == str(COMMENT_ID)
|
||||
assert captured["comment_id"] == COMMENT_ID
|
||||
assert captured["comment_like_count"] == "5"
|
||||
assert captured["sub_comment_count"] == "3"
|
||||
assert captured["parent_comment_id"] == "parent_abc"
|
||||
@@ -218,7 +218,7 @@ def test_weibo_store_end_to_end_sqlite():
|
||||
wb_, orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(wb.update_weibo_note(make_mock_note()))
|
||||
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
|
||||
asyncio.run(wb.update_weibo_note_comment(NOTE_ID, make_mock_comment()))
|
||||
finally:
|
||||
_restore(wb_, orig)
|
||||
|
||||
@@ -250,11 +250,8 @@ def test_weibo_store_end_to_end_sqlite():
|
||||
# 确认没有 desc 列存任何用户描述
|
||||
assert "desc" not in note_cols
|
||||
|
||||
# ---- 4. comment:同上。注意 store_comment 实现会把 comment_id/note_id/create_time
|
||||
# 转成 int 后再构造 ORM,此处忠实复刻该转换以走通 BigInteger 列写入 ----
|
||||
# ---- 4. comment:同上。ID 已统一为字符串类型,create_time 保持 int ----
|
||||
cc = dict(captured_comment)
|
||||
cc["comment_id"] = int(cc["comment_id"])
|
||||
cc["note_id"] = int(cc.get("note_id", 0) or 0)
|
||||
cc["create_time"] = int(cc.get("create_time", 0) or 0)
|
||||
comment_obj = WeiboNoteComment(**cc) # 不抛异常 => key 与 ORM 列对得上
|
||||
session.add(comment_obj)
|
||||
|
||||
Reference in New Issue
Block a user