mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-07-27 23:10:27 +08:00
refactor: 教学版移除全平台用户个人信息采集与持久化
- 用户 ID 转为匿名 creator_hash,昵称中间脱敏,IP/头像/主页/签名/性别不再采集 - 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台 - 删除 7 张 creator 档案 ORM 表,15 张内容/评论表新增 creator_hash 列 - B 站禁用粉丝/关注/联系人列表抓取 - 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试 测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed)
This commit is contained in:
362
tests/test_douyin_no_user_info.py
Normal file
362
tests/test_douyin_no_user_info.py
Normal file
@@ -0,0 +1,362 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
教学版回归测试(抖音 douyin):确保 store/douyin 存储链路不再持久化可定位真人的
|
||||
用户个人信息。
|
||||
|
||||
教学版约定:用户 ID(uid/sec_uid/short_user_id/user_unique_id)/IP/头像/签名/性别
|
||||
一律不持久化;原始 uid 经 tools.user_hash.anonymize_user_id 转成 creator_hash
|
||||
(sha256 截断 16 位)写入;昵称保留但经 mask_nickname 中间脱敏。desc(作品描述)是
|
||||
内容字段,保留。
|
||||
|
||||
覆盖:
|
||||
1. test_douyin_aweme_masks_user_info
|
||||
- mock aweme_item(含 author.uid/sec_uid/short_id/unique_id/nickname/
|
||||
avatar_thumb.url_list/signature、aweme_item.ip_label、statistics、video/
|
||||
music/images 等内容字段)
|
||||
- FakeStore 捕获 update_douyin_aweme 产出的存储 dict
|
||||
- 断言:不含禁用键;含 creator_hash(≠原 uid);nickname 脱敏≠原文;
|
||||
原始敏感值不在任何存储值中;内容字段正常提取
|
||||
2. test_douyin_comment_masks_user_info
|
||||
- 同上,针对 update_dy_aweme_comment(含嵌套 user.* / comment_item.ip_label /
|
||||
cid / text / image_list 等)
|
||||
3. test_douyin_store_end_to_end_sqlite
|
||||
- FakeStore 捕获真实 dict → DouyinAweme(**captured) 构造(ORM 列校验:
|
||||
dict 多了已删列会 TypeError)
|
||||
- 端到端:monkeypatch db_session 为内存 SQLite engine + create_all,
|
||||
走真实 DouyinDbStoreImplement.store_content(含 int(aweme_id) / if title
|
||||
判断 / DouyinAweme(**content_item)),查询回读,验证 dict key 与删列后 ORM
|
||||
完全对得上
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
import config
|
||||
import store.douyin as ds
|
||||
from database import db_session
|
||||
from database.models import Base, DouyinAweme, DouyinAwemeComment
|
||||
from tools.user_hash import anonymize_user_id, mask_nickname
|
||||
|
||||
|
||||
# 抖音教学版禁用字段(键):不得作为存储 dict 的 key 出现。
|
||||
FORBIDDEN_KEYS = {
|
||||
"user_id", "sec_uid", "short_user_id", "user_unique_id",
|
||||
"avatar", "user_signature", "ip_location",
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------- mock payload -----------------------------
|
||||
|
||||
def _build_aweme_item() -> dict:
|
||||
"""贴近真实抖音结构的 mock aweme_item。
|
||||
含会被拍平的用户字段(uid/sec_uid/short_id/unique_id/avatar/signature)与
|
||||
ip_label,这些必须不落库;同时含 statistics/video/music/images 等内容字段,
|
||||
让 _extract_* 辅助函数返回非空值。"""
|
||||
return {
|
||||
"aweme_id": "7234567890123456",
|
||||
"aweme_type": 0,
|
||||
"desc": "这是一个作品描述,同时作为 title",
|
||||
"create_time": 1700000000,
|
||||
"ip_label": "上海", # IP 归属地,禁用
|
||||
"author": {
|
||||
"uid": "9876543210",
|
||||
"sec_uid": "MS4wLjABAAAASecretSecUidForTest",
|
||||
"short_id": "88877766",
|
||||
"unique_id": "creator_unique_abc",
|
||||
"nickname": "抖音创作者",
|
||||
"avatar_thumb": {"url_list": ["http://x/avatar_thumb.jpg"]},
|
||||
"avatar_medium": {"url_list": ["http://x/avatar_medium.jpg"]},
|
||||
"signature": "这是创作者个人签名内容",
|
||||
},
|
||||
"statistics": {
|
||||
"digg_count": 100,
|
||||
"collect_count": 5,
|
||||
"comment_count": 20,
|
||||
"share_count": 3,
|
||||
},
|
||||
"video": {
|
||||
"raw_cover": {"url_list": ["", "http://x/cover.jpg"]},
|
||||
"origin_cover": {"url_list": ["", "http://x/cover2.jpg"]},
|
||||
"play_addr_h264": {"url_list": ["", "http://x/video_h264.mp4"]},
|
||||
"play_addr_256": {"url_list": ["", "http://x/video_256.mp4"]},
|
||||
"play_addr": {"url_list": ["", "http://x/video.mp4"]},
|
||||
},
|
||||
"music": {
|
||||
"play_url": {"uri": "http://x/music.mp3"},
|
||||
},
|
||||
"images": [
|
||||
{"url_list": ["", "http://x/note_img1.jpg"]},
|
||||
{"url_list": ["", "http://x/note_img2.jpg"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _build_comment_item() -> dict:
|
||||
"""贴近真实抖音结构的 mock comment_item。aweme_id 须与传入
|
||||
update_dy_aweme_comment 的 aweme_id 一致。"""
|
||||
return {
|
||||
"aweme_id": "7234567890123456",
|
||||
"cid": "1111111111",
|
||||
"text": "这是一条评论内容",
|
||||
"create_time": 1700000001,
|
||||
"reply_id": "0",
|
||||
"reply_comment_total": 2,
|
||||
"digg_count": 5,
|
||||
"ip_label": "北京", # IP 归属地,禁用
|
||||
"user": {
|
||||
"uid": "555666777",
|
||||
"sec_uid": "MS4wLjABBBBCommentSecUid",
|
||||
"short_id": "22233344",
|
||||
"unique_id": "commenter_unique_xyz",
|
||||
"nickname": "评论员小王",
|
||||
"avatar_medium": {"url_list": ["http://x/cavatar.jpg"]},
|
||||
"signature": "评论员个人签名内容",
|
||||
},
|
||||
"image_list": [
|
||||
{"origin_url": {"url_list": ["", "http://x/cimg.jpg"]}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------- 辅助 -----------------------------
|
||||
|
||||
class _FakeStore:
|
||||
"""捕获存储 dict,不真正落库。"""
|
||||
|
||||
def __init__(self):
|
||||
self.contents = []
|
||||
self.comments = []
|
||||
|
||||
async def store_content(self, content_item):
|
||||
self.contents.append(dict(content_item))
|
||||
|
||||
async def store_comment(self, comment_item):
|
||||
self.comments.append(dict(comment_item))
|
||||
|
||||
|
||||
def _patch_factory(fake):
|
||||
"""把 DouyinStoreFactory.create_store 替换为返回 fake,返回原方法用于还原。"""
|
||||
orig = ds.DouyinStoreFactory.create_store
|
||||
ds.DouyinStoreFactory.create_store = staticmethod(lambda: fake)
|
||||
return orig
|
||||
|
||||
|
||||
def _assert_no_forbidden(captured: dict, label: str):
|
||||
hit = set(captured.keys()) & FORBIDDEN_KEYS
|
||||
assert not hit, f"[{label}] 存储 dict 仍含禁用键: {hit}"
|
||||
|
||||
|
||||
def _assert_raw_values_absent(captured: dict, raw_values, label: str):
|
||||
"""禁用的原始敏感值(uid/sec_uid/头像/签名/IP 等)不得出现在任何存储值里。
|
||||
creator_hash 是由 uid 派生的匿名哈希(已单独断言 ≠ 原文),不参与子串扫描。"""
|
||||
leaked = []
|
||||
for rv in raw_values:
|
||||
if not rv:
|
||||
continue
|
||||
for k, v in captured.items():
|
||||
if k == "creator_hash":
|
||||
continue
|
||||
if isinstance(v, str) and rv in v:
|
||||
leaked.append((k, rv))
|
||||
assert not leaked, f"[{label}] 存储 dict 中泄漏了原始敏感值: {leaked}"
|
||||
|
||||
|
||||
# ----------------------------- 测试 -----------------------------
|
||||
|
||||
def test_douyin_aweme_masks_user_info():
|
||||
aweme = _build_aweme_item()
|
||||
raw_uid = aweme["author"]["uid"]
|
||||
raw_nick = aweme["author"]["nickname"]
|
||||
raw_sensitive = [
|
||||
raw_uid,
|
||||
aweme["author"]["sec_uid"],
|
||||
aweme["author"]["short_id"],
|
||||
aweme["author"]["unique_id"],
|
||||
aweme["author"]["avatar_thumb"]["url_list"][0],
|
||||
aweme["author"]["signature"],
|
||||
aweme["ip_label"],
|
||||
]
|
||||
|
||||
fake = _FakeStore()
|
||||
orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(ds.update_douyin_aweme(aweme))
|
||||
finally:
|
||||
ds.DouyinStoreFactory.create_store = orig
|
||||
|
||||
assert len(fake.contents) == 1
|
||||
captured = fake.contents[0]
|
||||
|
||||
# 1. 禁用键不出现
|
||||
_assert_no_forbidden(captured, "douyin_aweme")
|
||||
# 2. 原始敏感值不泄漏到任何存储值
|
||||
_assert_raw_values_absent(captured, raw_sensitive, "douyin_aweme")
|
||||
# 3. creator_hash 存在、≠原 uid、与 anonymize_user_id 一致
|
||||
assert captured.get("creator_hash")
|
||||
assert captured["creator_hash"] != raw_uid
|
||||
assert captured["creator_hash"] == anonymize_user_id(raw_uid)
|
||||
# 4. nickname 保留但脱敏,≠原文,与 mask_nickname 一致
|
||||
assert captured.get("nickname")
|
||||
assert captured["nickname"] != raw_nick
|
||||
assert captured["nickname"] == mask_nickname(raw_nick)
|
||||
# 5. 内容字段保留(desc/title 是作品描述,不禁用)
|
||||
assert captured.get("desc") == aweme["desc"]
|
||||
assert captured.get("title") == aweme["desc"]
|
||||
# 6. _extract_* 内容字段正常提取(非空)
|
||||
assert captured.get("cover_url") == "http://x/cover.jpg"
|
||||
assert captured.get("video_download_url") == "http://x/video_h264.mp4"
|
||||
assert captured.get("music_download_url") == "http://x/music.mp3"
|
||||
assert "http://x/note_img1.jpg" in captured.get("note_download_url", "")
|
||||
# 7. 互动数据拍平
|
||||
assert captured.get("liked_count") == "100"
|
||||
assert captured.get("collected_count") == "5"
|
||||
assert captured.get("comment_count") == "20"
|
||||
assert captured.get("share_count") == "3"
|
||||
assert captured.get("aweme_url") == f"https://www.douyin.com/video/{aweme['aweme_id']}"
|
||||
|
||||
|
||||
def test_douyin_comment_masks_user_info():
|
||||
aweme_id = "7234567890123456"
|
||||
comment = _build_comment_item()
|
||||
raw_uid = comment["user"]["uid"]
|
||||
raw_nick = comment["user"]["nickname"]
|
||||
raw_sensitive = [
|
||||
raw_uid,
|
||||
comment["user"]["sec_uid"],
|
||||
comment["user"]["short_id"],
|
||||
comment["user"]["unique_id"],
|
||||
comment["user"]["avatar_medium"]["url_list"][0],
|
||||
comment["user"]["signature"],
|
||||
comment["ip_label"],
|
||||
]
|
||||
|
||||
fake = _FakeStore()
|
||||
orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(ds.update_dy_aweme_comment(aweme_id, comment))
|
||||
finally:
|
||||
ds.DouyinStoreFactory.create_store = orig
|
||||
|
||||
assert len(fake.comments) == 1
|
||||
captured = fake.comments[0]
|
||||
|
||||
_assert_no_forbidden(captured, "douyin_comment")
|
||||
_assert_raw_values_absent(captured, raw_sensitive, "douyin_comment")
|
||||
|
||||
assert captured.get("creator_hash")
|
||||
assert captured["creator_hash"] != raw_uid
|
||||
assert captured["creator_hash"] == anonymize_user_id(raw_uid)
|
||||
assert captured.get("nickname")
|
||||
assert captured["nickname"] != raw_nick
|
||||
assert captured["nickname"] == mask_nickname(raw_nick)
|
||||
|
||||
# 评论内容/ID 保留
|
||||
assert captured.get("content") == comment["text"]
|
||||
assert captured.get("comment_id") == comment["cid"]
|
||||
assert captured.get("aweme_id") == aweme_id
|
||||
assert captured.get("parent_comment_id") == "0"
|
||||
assert captured.get("sub_comment_count") == "2"
|
||||
# 评论图片提取
|
||||
assert captured.get("pictures") == "http://x/cimg.jpg"
|
||||
|
||||
|
||||
def test_douyin_store_end_to_end_sqlite(monkeypatch):
|
||||
aweme = _build_aweme_item()
|
||||
raw_uid = aweme["author"]["uid"]
|
||||
raw_nick = aweme["author"]["nickname"]
|
||||
|
||||
# ---- 1. FakeStore 捕获 update_douyin_aweme 产出的真实 dict ----
|
||||
fake = _FakeStore()
|
||||
orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(ds.update_douyin_aweme(aweme))
|
||||
finally:
|
||||
ds.DouyinStoreFactory.create_store = orig
|
||||
assert len(fake.contents) == 1
|
||||
captured = fake.contents[0]
|
||||
|
||||
# ---- 2. DouyinAweme(**captured) 构造校验 ----
|
||||
# dict 多了已删列会 TypeError,少了非空必填列 SQLAlchemy 也会报错;
|
||||
# 此处证明 captured 的 key 与删列后 ORM 列完全对得上,不抛异常。
|
||||
obj = DouyinAweme(**captured)
|
||||
assert int(obj.aweme_id) == int(aweme["aweme_id"])
|
||||
assert obj.creator_hash == anonymize_user_id(raw_uid)
|
||||
assert obj.creator_hash != raw_uid
|
||||
assert obj.nickname == mask_nickname(raw_nick)
|
||||
assert obj.nickname != raw_nick
|
||||
assert obj.title == aweme["desc"]
|
||||
assert obj.desc == aweme["desc"]
|
||||
_assert_no_forbidden(captured, "douyin_aweme_orm_construct")
|
||||
|
||||
# ---- 3. 端到端:内存 SQLite + 真实 store_content ----
|
||||
# 用 StaticPool 保证 :memory: 库在同一个连接上持久(跨 session 可见)。
|
||||
engine = create_async_engine(
|
||||
"sqlite+aiosqlite:///:memory:",
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
# 让 db_session 用内存 engine;SAVE_DATA_OPTION=db 让工厂走 DouyinDbStoreImplement
|
||||
monkeypatch.setattr(db_session, "get_async_engine", lambda *a, **kw: engine)
|
||||
monkeypatch.setattr(config, "SAVE_DATA_OPTION", "db")
|
||||
|
||||
async def _scenario():
|
||||
# 建表
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
# 真实 store_content 路径(含 int(aweme_id) 与 if content_item.get("title") 判断)
|
||||
await ds.update_douyin_aweme(aweme)
|
||||
# 查询回读
|
||||
async with db_session.get_session() as session:
|
||||
res = await session.execute(
|
||||
select(DouyinAweme).where(DouyinAweme.aweme_id == int(aweme["aweme_id"]))
|
||||
)
|
||||
row = res.scalar_one_or_none()
|
||||
await engine.dispose()
|
||||
return row
|
||||
|
||||
row = asyncio.run(_scenario())
|
||||
|
||||
# ---- 4. 回读断言 ----
|
||||
assert row is not None, "作品未写入 SQLite"
|
||||
assert int(row.aweme_id) == int(aweme["aweme_id"])
|
||||
assert row.creator_hash == anonymize_user_id(raw_uid)
|
||||
assert row.creator_hash != raw_uid
|
||||
assert row.nickname == mask_nickname(raw_nick)
|
||||
assert row.nickname != raw_nick
|
||||
assert row.desc == aweme["desc"]
|
||||
assert row.title == aweme["desc"]
|
||||
assert row.cover_url == "http://x/cover.jpg"
|
||||
assert row.video_download_url == "http://x/video_h264.mp4"
|
||||
assert row.music_download_url == "http://x/music.mp3"
|
||||
# 禁用列在 ORM 上不存在(自省)
|
||||
orm_cols = {c.name for c in DouyinAweme.__table__.columns}
|
||||
assert not (orm_cols & FORBIDDEN_KEYS), f"ORM 仍含禁用列: {orm_cols & FORBIDDEN_KEYS}"
|
||||
# captured 的所有 key 都是合法 ORM 列(无悬空 key)
|
||||
assert set(captured.keys()).issubset(orm_cols), (
|
||||
f"captured 含非 ORM 列: {set(captured.keys()) - orm_cols}"
|
||||
)
|
||||
|
||||
# 评论同样做一次 ORM 构造校验(证明 comment dict key 对得上)
|
||||
comment = _build_comment_item()
|
||||
fake_c = _FakeStore()
|
||||
orig_c = _patch_factory(fake_c)
|
||||
try:
|
||||
asyncio.run(ds.update_dy_aweme_comment(comment["aweme_id"], comment))
|
||||
finally:
|
||||
ds.DouyinStoreFactory.create_store = orig_c
|
||||
captured_comment = fake_c.comments[0]
|
||||
comment_obj = DouyinAwemeComment(**captured_comment) # 不抛异常即对得上
|
||||
assert int(comment_obj.comment_id) == int(comment["cid"])
|
||||
assert comment_obj.creator_hash == anonymize_user_id(comment["user"]["uid"])
|
||||
assert comment_obj.nickname == mask_nickname(comment["user"]["nickname"])
|
||||
_assert_no_forbidden(captured_comment, "douyin_comment_orm_construct")
|
||||
comment_orm_cols = {c.name for c in DouyinAwemeComment.__table__.columns}
|
||||
assert set(captured_comment.keys()).issubset(comment_orm_cols), (
|
||||
f"captured_comment 含非 ORM 列: {set(captured_comment.keys()) - comment_orm_cols}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
284
tests/test_kuaishou_no_user_info.py
Normal file
284
tests/test_kuaishou_no_user_info.py
Normal file
@@ -0,0 +1,284 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
教学版回归测试:快手(kuaishou)存储链路不再持久化可定位真人的用户个人信息。
|
||||
|
||||
覆盖:
|
||||
1. update_kuaishou_video —— mock video_item(含 author.id/name/headerUrl + photo 内容字段 + type)
|
||||
经 FakeStore 捕获,断言捕获 dict 不含禁用键(user_id/avatar/signature/ip_location/gender)、
|
||||
含 creator_hash(≠原 user_id)、nickname 脱敏(≠原文)。
|
||||
2. update_ks_video_comment —— V2(snake_case) 与旧 GraphQL(camelCase) 两种 comment_item 格式各测一次。
|
||||
3. test_kuaishou_store_end_to_end_sqlite —— 端到端:FakeStore 捕获真实 dict ->
|
||||
KuaishouVideo(**captured) 触发 ORM 列校验 -> 写入内存 SQLite -> 查询回读校验属性。
|
||||
|
||||
约束:只新建本测试文件,只读源码不改源码;不依赖网络/登录。
|
||||
"""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
import store.kuaishou as ks
|
||||
from store.kuaishou import update_kuaishou_video, update_ks_video_comment
|
||||
from tools.user_hash import anonymize_user_id, mask_nickname
|
||||
|
||||
# 教学版禁用字段(键):一律不得出现在存储 dict 中。
|
||||
# 昵称字段 nickname 允许保留,但值须脱敏。
|
||||
FORBIDDEN_KEYS = {"user_id", "avatar", "signature", "ip_location", "gender"}
|
||||
|
||||
# ----------------------------- mock 数据 -----------------------------
|
||||
|
||||
MOCK_VIDEO_ID = "3xf8e9kq2b7w4"
|
||||
MOCK_AUTHOR_ID = "ks_author_001"
|
||||
MOCK_AUTHOR_NAME = "快手达人"
|
||||
MOCK_CAPTION = "这是一条测试视频,教学版脱敏回归 #测试"
|
||||
|
||||
# 评论作者(两种格式用不同 id/昵称,便于分别断言)
|
||||
MOCK_COMMENT_V2_AUTHOR_ID = "ks_user_888"
|
||||
MOCK_COMMENT_V2_AUTHOR_NAME = "快手老铁"
|
||||
MOCK_COMMENT_LEGACY_AUTHOR_ID = "ks_user_777"
|
||||
MOCK_COMMENT_LEGACY_AUTHOR_NAME = "快乐源泉"
|
||||
|
||||
|
||||
def make_mock_video() -> dict:
|
||||
"""贴近真实快手结构的 video_item:author 在顶层,photo 含内容字段。
|
||||
author.headerUrl 与各禁用字段一样不应进入存储 dict。"""
|
||||
return {
|
||||
"type": 1,
|
||||
"photo": {
|
||||
"id": MOCK_VIDEO_ID,
|
||||
"caption": MOCK_CAPTION,
|
||||
"timestamp": 1700000000000,
|
||||
"coverUrl": "https://p.kuaishou.com/cover/abc.jpg",
|
||||
"photoUrl": "https://v.kuaishou.com/play/abc.mp4",
|
||||
"realLikeCount": 12345,
|
||||
"viewCount": 67890,
|
||||
},
|
||||
"author": {
|
||||
"id": MOCK_AUTHOR_ID,
|
||||
"name": MOCK_AUTHOR_NAME,
|
||||
"headerUrl": "https://p.kuaishou.com/header/u001.jpg",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def make_mock_comment_v2() -> dict:
|
||||
"""V2 API 格式:snake_case 字段名,comment_id 为 int。"""
|
||||
return {
|
||||
"comment_id": 9001,
|
||||
"timestamp": 1700000001234,
|
||||
"content": "太搞笑了哈哈哈",
|
||||
"author_id": MOCK_COMMENT_V2_AUTHOR_ID,
|
||||
"author_name": MOCK_COMMENT_V2_AUTHOR_NAME,
|
||||
"headurl": "https://p.kuaishou.com/header/u888.jpg",
|
||||
"commentCount": 7,
|
||||
}
|
||||
|
||||
|
||||
def make_mock_comment_legacy() -> dict:
|
||||
"""旧 GraphQL API 格式:camelCase 字段名。"""
|
||||
return {
|
||||
"commentId": 8001,
|
||||
"timestamp": 1700000005678,
|
||||
"content": "这条评论来自旧 GraphQL 接口",
|
||||
"authorId": MOCK_COMMENT_LEGACY_AUTHOR_ID,
|
||||
"authorName": MOCK_COMMENT_LEGACY_AUTHOR_NAME,
|
||||
"headurl": "https://p.kuaishou.com/header/u777.jpg",
|
||||
"subCommentCount": 3,
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------- FakeStore 捕获 -----------------------------
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _patch_create_store():
|
||||
"""把 KuaishouStoreFactory.create_store 替换为返回捕获用 FakeStore 的 staticmethod。
|
||||
FakeStore 把 store_content/store_comment 收到的 dict 原样写入 holder.content / holder.comment。
|
||||
|
||||
保存/还原走类 __dict__ 中的 staticmethod 描述符本身,避免把 staticmethod 退化为普通方法
|
||||
而污染后续测试。"""
|
||||
holder = types.SimpleNamespace(content={}, comment={})
|
||||
|
||||
class FakeStore:
|
||||
async def store_content(self, content_item):
|
||||
holder.content.clear()
|
||||
holder.content.update(content_item)
|
||||
|
||||
async def store_comment(self, comment_item):
|
||||
holder.comment.clear()
|
||||
holder.comment.update(comment_item)
|
||||
|
||||
async def store_creator(self, creator):
|
||||
pass
|
||||
|
||||
orig = ks.KuaishouStoreFactory.__dict__["create_store"]
|
||||
ks.KuaishouStoreFactory.create_store = staticmethod(lambda: FakeStore())
|
||||
try:
|
||||
yield holder
|
||||
finally:
|
||||
ks.KuaishouStoreFactory.create_store = orig
|
||||
|
||||
|
||||
def _assert_no_forbidden_keys(d: dict, label: str):
|
||||
hit = set(d.keys()) & FORBIDDEN_KEYS
|
||||
assert not hit, f"[{label}] 输出仍含禁用字段键: {hit}"
|
||||
|
||||
|
||||
# ----------------------------- 测试 -----------------------------
|
||||
|
||||
|
||||
def test_kuaishou_video_masks_user_info():
|
||||
"""video 链路:原始 user_id 转 creator_hash、昵称脱敏、headerUrl/禁用键不落库。"""
|
||||
with _patch_create_store() as holder:
|
||||
asyncio.run(update_kuaishou_video(make_mock_video()))
|
||||
captured = holder.content
|
||||
|
||||
assert captured, "FakeStore 未捕获到 content_item"
|
||||
_assert_no_forbidden_keys(captured, "kuaishou_video")
|
||||
# 头像字段不应进入存储 dict(author.headerUrl 已被丢弃)
|
||||
assert "headerUrl" not in captured and "avatar" not in captured
|
||||
|
||||
# creator_hash 存在且不等于原始 user_id
|
||||
assert captured.get("creator_hash") == anonymize_user_id(MOCK_AUTHOR_ID)
|
||||
assert captured["creator_hash"] != MOCK_AUTHOR_ID
|
||||
assert captured["creator_hash"] # 非空
|
||||
|
||||
# 昵称已脱敏:等于 mask_nickname(原文) 且不等于原文
|
||||
assert captured.get("nickname") == mask_nickname(MOCK_AUTHOR_NAME)
|
||||
assert captured["nickname"] != MOCK_AUTHOR_NAME
|
||||
assert "*" in captured["nickname"]
|
||||
|
||||
# 内容字段保留(video_id / desc / title)
|
||||
assert captured["video_id"] == MOCK_VIDEO_ID
|
||||
assert captured["desc"] == MOCK_CAPTION
|
||||
assert captured["title"] == MOCK_CAPTION
|
||||
# video_type 来自 video_item.type,被 str() 化
|
||||
assert captured["video_type"] == "1"
|
||||
# 计数字段被 str() 化
|
||||
assert captured["liked_count"] == "12345"
|
||||
assert captured["viewd_count"] == "67890"
|
||||
|
||||
|
||||
def test_kuaishou_comment_v2_masks_user_info():
|
||||
"""评论 V2(snake_case)格式:author_id/author_name 经匿名+脱敏,headurl 不落库。"""
|
||||
with _patch_create_store() as holder:
|
||||
asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_v2()))
|
||||
captured = holder.comment
|
||||
|
||||
assert captured, "FakeStore 未捕获到 comment_item"
|
||||
_assert_no_forbidden_keys(captured, "kuaishou_comment_v2")
|
||||
assert "headurl" not in captured and "avatar" not in captured
|
||||
|
||||
# comment_id 由 int 转为 str
|
||||
assert captured["comment_id"] == "9001"
|
||||
assert captured["video_id"] == MOCK_VIDEO_ID
|
||||
assert captured["content"] == "太搞笑了哈哈哈"
|
||||
# V2 用 commentCount
|
||||
assert captured["sub_comment_count"] == "7"
|
||||
|
||||
# creator_hash / 昵称
|
||||
assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_V2_AUTHOR_ID)
|
||||
assert captured["creator_hash"] != MOCK_COMMENT_V2_AUTHOR_ID
|
||||
assert captured["nickname"] == mask_nickname(MOCK_COMMENT_V2_AUTHOR_NAME)
|
||||
assert captured["nickname"] != MOCK_COMMENT_V2_AUTHOR_NAME
|
||||
assert "*" in captured["nickname"]
|
||||
|
||||
|
||||
def test_kuaishou_comment_legacy_masks_user_info():
|
||||
"""评论旧 GraphQL(camelCase)格式:authorId/authorName 经匿名+脱敏,headurl 不落库。"""
|
||||
with _patch_create_store() as holder:
|
||||
asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_legacy()))
|
||||
captured = holder.comment
|
||||
|
||||
assert captured, "FakeStore 未捕获到 comment_item"
|
||||
_assert_no_forbidden_keys(captured, "kuaishou_comment_legacy")
|
||||
assert "headurl" not in captured and "avatar" not in captured
|
||||
|
||||
# commentId 由 int 转为 str
|
||||
assert captured["comment_id"] == "8001"
|
||||
assert captured["video_id"] == MOCK_VIDEO_ID
|
||||
assert captured["content"] == "这条评论来自旧 GraphQL 接口"
|
||||
# 旧格式用 subCommentCount
|
||||
assert captured["sub_comment_count"] == "3"
|
||||
|
||||
# creator_hash / 昵称
|
||||
assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_LEGACY_AUTHOR_ID)
|
||||
assert captured["creator_hash"] != MOCK_COMMENT_LEGACY_AUTHOR_ID
|
||||
assert captured["nickname"] == mask_nickname(MOCK_COMMENT_LEGACY_AUTHOR_NAME)
|
||||
assert captured["nickname"] != MOCK_COMMENT_LEGACY_AUTHOR_NAME
|
||||
assert "*" in captured["nickname"]
|
||||
|
||||
|
||||
def test_kuaishou_store_end_to_end_sqlite():
|
||||
"""端到端:FakeStore 捕获真实 dict -> KuaishouVideo(**captured) ORM 列校验
|
||||
-> 写入内存 SQLite -> 查询回读,验证属性正确且无禁用列。
|
||||
|
||||
全程在同一个事件循环内完成(aiosqlite 连接绑定事件循环,跨 loop 会报错),
|
||||
不 patch db_session.get_session,而是直接用自建内存 engine 走 ORM 写入/查询。"""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from database.models import Base, KuaishouVideo, KuaishouVideoComment
|
||||
|
||||
async def run():
|
||||
# 内存 SQLite + StaticPool:单连接共享,保证 create_all 与后续读写同一库
|
||||
engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool)
|
||||
SessionFactory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(
|
||||
Base.metadata.create_all,
|
||||
tables=[KuaishouVideo.__table__, KuaishouVideoComment.__table__],
|
||||
)
|
||||
|
||||
# 1) 经 update_kuaishou_video 产生真实存储 dict(FakeStore 捕获)
|
||||
with _patch_create_store() as holder:
|
||||
await update_kuaishou_video(make_mock_video())
|
||||
captured = holder.content
|
||||
assert captured, "FakeStore 未捕获到 content_item"
|
||||
|
||||
# 2) ORM 列校验:captured 的所有 key 必须是 KuaishouVideo 的合法列,
|
||||
# 否则 KuaishouVideo(**captured) 抛 TypeError(若有禁用/多余键即暴露源码 bug)
|
||||
valid_cols = {c.name for c in KuaishouVideo.__table__.columns}
|
||||
assert set(captured.keys()) <= valid_cols, (
|
||||
f"captured 含非合法列: {set(captured.keys()) - valid_cols}"
|
||||
)
|
||||
obj = KuaishouVideo(**captured) # 不抛异常即通过列校验
|
||||
assert obj.video_id == MOCK_VIDEO_ID
|
||||
assert obj.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID)
|
||||
assert obj.creator_hash != MOCK_AUTHOR_ID
|
||||
assert obj.nickname == mask_nickname(MOCK_AUTHOR_NAME)
|
||||
assert obj.nickname != MOCK_AUTHOR_NAME
|
||||
assert obj.desc == MOCK_CAPTION # 内容保留
|
||||
|
||||
# 3) 真实写库 + 查询回读
|
||||
async with SessionFactory() as session:
|
||||
session.add(obj)
|
||||
await session.commit()
|
||||
|
||||
res = await session.execute(
|
||||
select(KuaishouVideo).where(KuaishouVideo.video_id == MOCK_VIDEO_ID)
|
||||
)
|
||||
row = res.scalar_one()
|
||||
assert row is not None
|
||||
assert row.video_id == MOCK_VIDEO_ID
|
||||
assert row.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID)
|
||||
assert row.creator_hash != MOCK_AUTHOR_ID
|
||||
assert row.nickname == mask_nickname(MOCK_AUTHOR_NAME)
|
||||
assert row.nickname != MOCK_AUTHOR_NAME
|
||||
# 内容字段保留
|
||||
assert row.desc == MOCK_CAPTION
|
||||
assert row.title == MOCK_CAPTION
|
||||
# ORM 行对象上不应存在任何禁用列属性
|
||||
for bad in FORBIDDEN_KEYS:
|
||||
assert not hasattr(row, bad), f"KuaishouVideo 行对象仍含禁用属性: {bad}"
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
239
tests/test_no_user_info.py
Normal file
239
tests/test_no_user_info.py
Normal file
@@ -0,0 +1,239 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
教学版回归测试:确保爬取/存储链路不再持久化可定位真人的用户个人信息。
|
||||
|
||||
覆盖:
|
||||
1. ORM 自省 —— database.models 中无禁用列、creator 档案表已删除、内容/评论表含 creator_hash。
|
||||
2. 提取层 —— 用 mock API/HTML payload 喂各平台提取器,断言输出 dict 不含禁用字段、
|
||||
不含原始 user_id、昵称已脱敏且不等于原文。
|
||||
3. 仓库 grep 断言 —— store/ 与 media_platform/ 不再把禁用字段作为存储 dict 的 key。
|
||||
"""
|
||||
import re
|
||||
import subprocess
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
|
||||
# 统一的禁用字段名(键)。昵称字段(nickname/user_nickname/screen_name/name/user_name)允许保留(值需脱敏)。
|
||||
FORBIDDEN_KEYS = {
|
||||
"user_id", "sec_uid", "short_user_id", "user_unique_id", "user_signature",
|
||||
"avatar", "user_avatar", "face", "sign", "profile_url", "user_link",
|
||||
"url_token", "user_url_token", "ip_location", "ip_address", "gender", "sex",
|
||||
"up_id", "fan_id", "up_name", "fan_name", "up_avatar", "fan_avatar",
|
||||
"up_sign", "fan_sign", "mid",
|
||||
}
|
||||
NICK_KEYS = {"nickname", "user_nickname", "screen_name", "name", "user_name"}
|
||||
MASK_RE = re.compile(r"^.?\*{1,4}.?$")
|
||||
|
||||
|
||||
# ----------------------------- ORM 自省 -----------------------------
|
||||
|
||||
def test_orm_has_no_forbidden_columns():
|
||||
import database.models as m
|
||||
from sqlalchemy.orm import class_mapper
|
||||
tables = [c for c in dir(m) if c[0].isupper()
|
||||
and c not in ("Base", "Column", "Integer", "BigInteger", "String", "Text")]
|
||||
bad = []
|
||||
for t in tables:
|
||||
cols = {c.name for c in class_mapper(getattr(m, t)).columns}
|
||||
hit = cols & FORBIDDEN_KEYS
|
||||
if hit:
|
||||
bad.append((t, sorted(hit)))
|
||||
assert not bad, f"ORM 仍含禁用列: {bad}"
|
||||
|
||||
|
||||
def test_creator_tables_removed():
|
||||
import database.models as m
|
||||
removed = {"XhsCreator", "DyCreator", "WeiboCreator", "TiebaCreator",
|
||||
"ZhihuCreator", "BilibiliUpInfo", "BilibiliContactInfo"}
|
||||
for t in removed:
|
||||
assert not hasattr(m, t), f"creator 档案表 {t} 仍存在"
|
||||
|
||||
|
||||
def test_content_tables_have_creator_hash():
|
||||
import database.models as m
|
||||
from sqlalchemy.orm import class_mapper
|
||||
content_tables = ["XhsNote", "XhsNoteComment", "WeiboNote", "WeiboNoteComment",
|
||||
"BilibiliVideo", "BilibiliVideoComment", "BilibiliUpDynamic",
|
||||
"DouyinAweme", "DouyinAwemeComment", "KuaishouVideo",
|
||||
"KuaishouVideoComment", "TiebaNote", "TiebaComment",
|
||||
"ZhihuContent", "ZhihuComment"]
|
||||
for t in content_tables:
|
||||
cols = {c.name for c in class_mapper(getattr(m, t)).columns}
|
||||
assert "creator_hash" in cols, f"{t} 缺少 creator_hash 列"
|
||||
|
||||
|
||||
# ----------------------------- 提取层 mock -----------------------------
|
||||
|
||||
def _check_no_forbidden_keys(d: dict, label: str):
|
||||
keys = set(d.keys())
|
||||
hit = keys & FORBIDDEN_KEYS
|
||||
assert not hit, f"[{label}] 输出仍含禁用字段键: {hit}"
|
||||
|
||||
|
||||
def _check_nickname_masked(d: dict, raw: str, label: str):
|
||||
nick_keys = set(d.keys()) & NICK_KEYS
|
||||
assert nick_keys, f"[{label}] 未保留任何昵称字段(应保留并脱敏)"
|
||||
for k in nick_keys:
|
||||
val = d[k]
|
||||
if val in ("", None):
|
||||
continue
|
||||
assert val != raw, f"[{label}] {k} 未脱敏,仍为原文: {val}"
|
||||
assert MASK_RE.match(val) or "*" in val, f"[{label}] {k} 未脱敏: {val}"
|
||||
|
||||
|
||||
def test_mask_and_hash_tools():
|
||||
from tools.user_hash import anonymize_user_id, mask_nickname
|
||||
h = anonymize_user_id("12345")
|
||||
assert h and h != "12345" and re.fullmatch(r"[0-9a-f]{16}", h)
|
||||
assert anonymize_user_id(None) == "" and anonymize_user_id("") == ""
|
||||
# 昵称脱敏:首尾留1字、中间星号,且不等于原文
|
||||
assert mask_nickname("张三丰") != "张三丰"
|
||||
assert "*" in mask_nickname("张三丰")
|
||||
assert mask_nickname(None) == ""
|
||||
assert mask_nickname("a") == "*"
|
||||
|
||||
|
||||
def test_xhs_note_extraction_masks_user_info():
|
||||
import asyncio
|
||||
import store.xhs as xs
|
||||
note_item = {
|
||||
"note_id": "abc",
|
||||
"type": "normal",
|
||||
"title": "t",
|
||||
"desc": "d",
|
||||
"time": 1,
|
||||
"last_update_time": 0,
|
||||
"user": {"user_id": "u123", "nickname": "小红同学", "avatar": "http://x/a.jpg"},
|
||||
"ip_location": "上海",
|
||||
"interact_info": {"liked_count": "1", "collected_count": "0",
|
||||
"comment_count": "0", "share_count": "0"},
|
||||
"image_list": [], "tag_list": [], "xsec_token": "tok",
|
||||
}
|
||||
captured = {}
|
||||
|
||||
class FakeStore:
|
||||
async def store_content(self, content_item):
|
||||
captured.update(content_item)
|
||||
|
||||
orig = xs.XhsStoreFactory.create_store
|
||||
xs.XhsStoreFactory.create_store = staticmethod(lambda: FakeStore())
|
||||
try:
|
||||
asyncio.run(xs.update_xhs_note(note_item))
|
||||
finally:
|
||||
xs.XhsStoreFactory.create_store = orig
|
||||
_check_no_forbidden_keys(captured, "xhs_note")
|
||||
assert captured.get("creator_hash") != "u123"
|
||||
_check_nickname_masked(captured, "小红同学", "xhs_note")
|
||||
|
||||
|
||||
def test_tieba_note_extraction_masks_user_info():
|
||||
from media_platform.tieba.help import TieBaExtractor
|
||||
api_data = {
|
||||
"thread": {"id": 1, "title": "tt", "reply_num": 5},
|
||||
"first_floor": {"tid": 1, "author_id": 9, "time": 1700000000, "content": "c"},
|
||||
"forum": {"name": "test", "id": 1},
|
||||
"page": {"total_page": 1},
|
||||
"user_list": [{"id": 9, "name_show": "贴吧老哥", "name": "lg", "portrait": "p", "ip_address": "北京"}],
|
||||
}
|
||||
note = TieBaExtractor().extract_note_detail_from_api(api_data)
|
||||
d = note.model_dump()
|
||||
_check_no_forbidden_keys(d, "tieba_note")
|
||||
assert d.get("creator_hash") # user_link 已转哈希
|
||||
_check_nickname_masked(d, "贴吧老哥", "tieba_note")
|
||||
|
||||
|
||||
def test_tieba_comment_extraction_masks_user_info():
|
||||
from media_platform.tieba.help import TieBaExtractor
|
||||
from model.m_baidu_tieba import TiebaNote
|
||||
api_data = {
|
||||
"forum": {"id": 1, "name": "test"},
|
||||
"post_list": [{"id": 7, "author_id": 9, "time": 1700000000, "content": "c", "sub_post_number": 0}],
|
||||
"user_list": [{"id": 9, "name_show": "评论员", "name": "py", "portrait": "p", "ip_address": "上海"}],
|
||||
}
|
||||
note_detail = TiebaNote(note_id="1", title="t", note_url="u", tieba_name="test", tieba_link="l")
|
||||
comments = TieBaExtractor().extract_tieba_note_parent_comments_from_api(api_data, note_detail)
|
||||
assert comments
|
||||
d = comments[0].model_dump()
|
||||
_check_no_forbidden_keys(d, "tieba_comment")
|
||||
_check_nickname_masked(d, "评论员", "tieba_comment")
|
||||
|
||||
|
||||
def test_zhihu_comment_extraction_masks_user_info():
|
||||
from media_platform.zhihu.help import ZhihuExtractor
|
||||
from model.m_zhihu import ZhihuContent
|
||||
comments_raw = [{
|
||||
"type": "comment", "id": 1, "content": "c", "created_time": 1700000000,
|
||||
"like_count": 1, "dislike_count": 0, "child_comment_count": 0,
|
||||
"author": {"id": "z9", "name": "知乎答主", "url_token": "tok", "avatar_url": "http://x/a.jpg"},
|
||||
"comment_tag": [{"type": "ip_info", "text": "广东"}],
|
||||
}]
|
||||
page_content = ZhihuContent(content_id="c1", content_type="answer")
|
||||
comments = ZhihuExtractor().extract_comments(page_content, comments_raw)
|
||||
assert comments
|
||||
d = comments[0].model_dump() if hasattr(comments[0], "model_dump") else vars(comments[0])
|
||||
_check_no_forbidden_keys(d, "zhihu_comment")
|
||||
assert "creator_hash" in d and d["creator_hash"]
|
||||
_check_nickname_masked(d, "知乎答主", "zhihu_comment")
|
||||
|
||||
|
||||
def test_bilibili_video_dict_masks_user_info():
|
||||
# 直接测 store/bilibili/__init__.py 的拍平逻辑(不触发网络)
|
||||
import asyncio
|
||||
from store.bilibili import update_bilibili_video
|
||||
video_item = {
|
||||
"View": {
|
||||
"aid": 100, "title": "t", "desc": "d", "pubdate": 1,
|
||||
"owner": {"mid": 777, "name": "UP主大人", "face": "http://x/a.jpg"},
|
||||
"stat": {"like": 1, "view": 2},
|
||||
"pic": "http://x/cover.jpg",
|
||||
}
|
||||
}
|
||||
# 拦截真实存储:替换工厂返回一个捕获 dict 的假 store
|
||||
captured = {}
|
||||
|
||||
class FakeStore:
|
||||
async def store_content(self, content_item):
|
||||
captured.update(content_item)
|
||||
|
||||
import store.bilibili as bs
|
||||
orig = bs.BiliStoreFactory.create_store
|
||||
bs.BiliStoreFactory.create_store = staticmethod(lambda: FakeStore())
|
||||
try:
|
||||
asyncio.get_event_loop().run_until_complete(update_bilibili_video(video_item)) \
|
||||
if False else asyncio.run(update_bilibili_video(video_item))
|
||||
finally:
|
||||
bs.BiliStoreFactory.create_store = orig
|
||||
_check_no_forbidden_keys(captured, "bili_video")
|
||||
assert captured.get("creator_hash") != 777
|
||||
_check_nickname_masked(captured, "UP主大人", "bili_video")
|
||||
|
||||
|
||||
# ----------------------------- 仓库 grep 断言 -----------------------------
|
||||
|
||||
def test_store_no_forbidden_dict_keys():
|
||||
# store/ 下不得把禁用字段作为存储 dict 的 key("field": value 形式)
|
||||
out = subprocess.run(
|
||||
["grep", "-rnE", '"(' + "|".join(FORBIDDEN_KEYS) + r')"\s*:', str(ROOT / "store")],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
# 允许的例外:Mongo store_creator 里的 query={"user_id": ...} 已全部改为 pass,应为空
|
||||
assert out.stdout.strip() == "", f"store/ 仍写入禁用字段键:\n{out.stdout}"
|
||||
|
||||
|
||||
def test_store_no_creator_orm_imports():
|
||||
# 已删除的 creator ORM 表(XhsCreator/DyCreator/...)不得再从 database.models 导入。
|
||||
# 注意:model/m_*.py 里的同名 pydantic 类是内存类型,允许保留。
|
||||
out = subprocess.run(
|
||||
["grep", "-rnE",
|
||||
r"from database\.models import.*(XhsCreator|DyCreator|WeiboCreator|TiebaCreator|ZhihuCreator|BilibiliUpInfo|BilibiliContactInfo)",
|
||||
str(ROOT / "store")],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert out.stdout.strip() == "", f"store/ 仍 import 已删除的 creator ORM 表:\n{out.stdout}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
278
tests/test_weibo_no_user_info.py
Normal file
278
tests/test_weibo_no_user_info.py
Normal file
@@ -0,0 +1,278 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
教学版回归测试(微博 weibo):确保微博存储链路不再持久化可定位真人的用户个人信息。
|
||||
|
||||
覆盖:
|
||||
1. test_weibo_note_masks_user_info —— 用贴近真实微博结构的 mock note_item 喂
|
||||
store.weibo.update_weibo_note,用 FakeStore 捕获拍平后的存储 dict,断言:
|
||||
- 不含任何禁用字段键(user_id/avatar/gender/profile_url/ip_location/desc ...)
|
||||
- 含 creator_hash,且不等于原始 user id
|
||||
- nickname 已脱敏且不等于原文
|
||||
2. test_weibo_comment_masks_user_info —— 同上,对 update_weibo_note_comment。
|
||||
3. test_weibo_store_end_to_end_sqlite —— 端到端:把 update_weibo_note /
|
||||
update_weibo_note_comment 产生的真实 dict 用 SQLite 内存库走完整 ORM
|
||||
写入+查询。WeiboNote(**captured_dict) 会因 SQLAlchemy 声明式构造器对未知
|
||||
关键字的校验,在 dict 含已删列时直接抛 TypeError —— 以此证明 dict 的 key
|
||||
与删列后的 ORM 完全对得上,且表中无禁用列、有 creator_hash。
|
||||
|
||||
说明:微博 note 的正文存于 content 字段,update_weibo_note 不产生 desc 键
|
||||
(WeiboNote ORM 亦无 desc 列),故不存在用户 description 被持久化的风险。
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
# 原始(明文)测试数据
|
||||
RAW_USER_ID = 7654321
|
||||
RAW_NICKNAME = "微博达人"
|
||||
RAW_COMMENT_USER_ID = 111222
|
||||
RAW_COMMENT_NICKNAME = "评论员小张"
|
||||
NOTE_ID = 5123456789
|
||||
COMMENT_ID = 998877
|
||||
# 合法 RFC2822 时间串(weekday 与日期已对齐:2025-06-14 是周六)
|
||||
RFC2822_TIME = "Sat Jun 14 12:00:00 +0800 2025"
|
||||
|
||||
# 禁用字段名(键)。昵称字段允许保留,但值必须脱敏。
|
||||
FORBIDDEN_KEYS = {
|
||||
"user_id", "sec_uid", "short_user_id", "user_unique_id",
|
||||
"avatar", "user_avatar", "face", "sign", "profile_url", "user_link",
|
||||
"ip_location", "ip_address", "gender", "sex", "desc",
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------- mock 数据 -----------------------------
|
||||
|
||||
def make_mock_note() -> dict:
|
||||
"""贴近真实 m.weibo.cn 接口结构的 mock note_item(含嵌套 user 信息)。"""
|
||||
return {
|
||||
"mblog": {
|
||||
"id": NOTE_ID,
|
||||
"text": "今天天气不错 <a href='#'>@好友</a> 出去玩",
|
||||
"created_at": RFC2822_TIME,
|
||||
"attitudes_count": 10,
|
||||
"comments_count": 2,
|
||||
"reposts_count": 1,
|
||||
"user": {
|
||||
"id": RAW_USER_ID,
|
||||
"screen_name": RAW_NICKNAME,
|
||||
"avatar_hd": "https://wx avatar.example.com/7654321.jpg",
|
||||
"gender": "f",
|
||||
"profile_url": "https://m.weibo.cn/profile/7654321",
|
||||
"description": "这是一个用户签名",
|
||||
"ip_location": "上海",
|
||||
"followers_count": 9999,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def make_mock_comment() -> dict:
|
||||
"""贴近真实微博评论结构的 mock comment_item(含嵌套 user 信息)。"""
|
||||
return {
|
||||
"id": COMMENT_ID,
|
||||
"text": "说得好 <a href='#'>支持</a>",
|
||||
"created_at": RFC2822_TIME,
|
||||
"total_number": 3,
|
||||
"like_count": 5,
|
||||
"rootid": "parent_abc",
|
||||
"user": {
|
||||
"id": RAW_COMMENT_USER_ID,
|
||||
"screen_name": RAW_COMMENT_NICKNAME,
|
||||
"avatar_hd": "https://wx avatar.example.com/111222.jpg",
|
||||
"gender": "m",
|
||||
"profile_url": "https://m.weibo.cn/profile/111222",
|
||||
"description": "评论员签名",
|
||||
"ip_location": "广东",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------- FakeStore 捕获 -----------------------------
|
||||
|
||||
class _FakeStore:
|
||||
"""捕获 store_content / store_comment 收到的 dict,不触发任何真实存储。"""
|
||||
|
||||
def __init__(self):
|
||||
self.captured_content = {}
|
||||
self.captured_comment = {}
|
||||
|
||||
async def store_content(self, content_item):
|
||||
self.captured_content.update(content_item)
|
||||
|
||||
async def store_comment(self, comment_item):
|
||||
self.captured_comment.update(comment_item)
|
||||
|
||||
async def store_creator(self, creator):
|
||||
pass
|
||||
|
||||
|
||||
def _patch_factory(fake: "_FakeStore"):
|
||||
"""把 store.weibo.WeibostoreFactory.create_store 替换为返回 fake 的静态方法,
|
||||
返回 (module, orig) 便于 finally 还原。"""
|
||||
import store.weibo as wb
|
||||
orig = wb.WeibostoreFactory.create_store
|
||||
wb.WeibostoreFactory.create_store = staticmethod(lambda: fake)
|
||||
return wb, orig
|
||||
|
||||
|
||||
def _restore(wb, orig):
|
||||
wb.WeibostoreFactory.create_store = orig
|
||||
|
||||
|
||||
# ----------------------------- 测试 -----------------------------
|
||||
|
||||
def test_weibo_note_masks_user_info():
|
||||
"""note 拍平后的存储 dict 不含禁用键、creator_hash 不等于原始 user id、昵称已脱敏。"""
|
||||
import store.weibo as wb
|
||||
|
||||
fake = _FakeStore()
|
||||
wb_, orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(wb.update_weibo_note(make_mock_note()))
|
||||
finally:
|
||||
_restore(wb_, orig)
|
||||
|
||||
captured = fake.captured_content
|
||||
assert captured, "FakeStore 未捕获到 note dict"
|
||||
|
||||
# 1. 不含任何禁用字段键
|
||||
hit = set(captured.keys()) & FORBIDDEN_KEYS
|
||||
assert not hit, f"note 存储 dict 仍含禁用字段键: {hit}"
|
||||
|
||||
# 2. creator_hash 存在、是 16 位 hex、不等于原始 user id
|
||||
creator_hash = captured.get("creator_hash")
|
||||
assert creator_hash, "note dict 缺少 creator_hash"
|
||||
assert creator_hash != str(RAW_USER_ID)
|
||||
assert creator_hash != RAW_USER_ID
|
||||
assert len(creator_hash) == 16
|
||||
|
||||
# 3. 昵称已脱敏:不等于原文且含星号
|
||||
nickname = captured.get("nickname")
|
||||
assert nickname, "note dict 缺少 nickname"
|
||||
assert nickname != RAW_NICKNAME, "note 昵称未脱敏,仍为原文"
|
||||
assert "*" in nickname, f"note 昵称未脱敏: {nickname}"
|
||||
|
||||
# 4. 内容字段正确(正文存于 content,不是 desc)
|
||||
assert "hello" not in captured # 确认没误存
|
||||
assert "今天天气不错" in captured["content"]
|
||||
assert captured["note_id"] == NOTE_ID
|
||||
assert captured["liked_count"] == "10"
|
||||
assert captured["comments_count"] == "2"
|
||||
assert captured["shared_count"] == "1"
|
||||
|
||||
|
||||
def test_weibo_comment_masks_user_info():
|
||||
"""comment 拍平后的存储 dict 不含禁用键、creator_hash 不等于原始 user id、昵称已脱敏。"""
|
||||
import store.weibo as wb
|
||||
|
||||
fake = _FakeStore()
|
||||
wb_, orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
|
||||
finally:
|
||||
_restore(wb_, orig)
|
||||
|
||||
captured = fake.captured_comment
|
||||
assert captured, "FakeStore 未捕获到 comment dict"
|
||||
|
||||
# 1. 不含任何禁用字段键
|
||||
hit = set(captured.keys()) & FORBIDDEN_KEYS
|
||||
assert not hit, f"comment 存储字典仍含禁用字段键: {hit}"
|
||||
|
||||
# 2. creator_hash 存在、不等于原始 user id
|
||||
creator_hash = captured.get("creator_hash")
|
||||
assert creator_hash, "comment dict 缺少 creator_hash"
|
||||
assert creator_hash != str(RAW_COMMENT_USER_ID)
|
||||
assert creator_hash != RAW_COMMENT_USER_ID
|
||||
assert len(creator_hash) == 16
|
||||
|
||||
# 3. 昵称已脱敏
|
||||
nickname = captured.get("nickname")
|
||||
assert nickname, "comment dict 缺少 nickname"
|
||||
assert nickname != RAW_COMMENT_NICKNAME, "comment 昵称未脱敏,仍为原文"
|
||||
assert "*" in nickname, f"comment 昵称未脱敏: {nickname}"
|
||||
|
||||
# 4. 内容字段正确
|
||||
assert "说得好" in captured["content"]
|
||||
assert captured["comment_id"] == str(COMMENT_ID)
|
||||
assert captured["comment_like_count"] == "5"
|
||||
assert captured["sub_comment_count"] == "3"
|
||||
assert captured["parent_comment_id"] == "parent_abc"
|
||||
|
||||
|
||||
def test_weibo_store_end_to_end_sqlite():
|
||||
"""端到端:捕获 note/comment 的真实 dict,用 SQLite 内存库走完整 ORM 写入+查询。
|
||||
|
||||
关键点:WeiboNote(**captured_dict) / WeiboNoteComment(**captured_dict) 会触发
|
||||
SQLAlchemy 声明式构造器的关键字校验——若 dict 含已删列(如 avatar/gender)会直接
|
||||
抛 TypeError。此处不抛异常即证明 dict 的 key 与删列后的 ORM 列完全对得上。
|
||||
"""
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
import store.weibo as wb
|
||||
from database.models import Base, WeiboNote, WeiboNoteComment
|
||||
|
||||
# ---- 1. 用 FakeStore 捕获 update_weibo_note / update_weibo_note_comment 产生的真实 dict ----
|
||||
fake = _FakeStore()
|
||||
wb_, orig = _patch_factory(fake)
|
||||
try:
|
||||
asyncio.run(wb.update_weibo_note(make_mock_note()))
|
||||
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
|
||||
finally:
|
||||
_restore(wb_, orig)
|
||||
|
||||
captured_note = dict(fake.captured_content)
|
||||
captured_comment = dict(fake.captured_comment)
|
||||
assert captured_note and captured_comment
|
||||
|
||||
# ---- 2. SQLite 内存库,建 weibo 两张表 ----
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(engine, tables=[WeiboNote.__table__, WeiboNoteComment.__table__])
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
try:
|
||||
# ---- 3. note:构造 ORM 对象(dict 多了已删列会直接 TypeError)并写入 ----
|
||||
note_obj = WeiboNote(**captured_note) # 不抛异常 => key 与 ORM 列对得上
|
||||
session.add(note_obj)
|
||||
session.commit()
|
||||
|
||||
row = session.query(WeiboNote).one()
|
||||
note_cols = {c.name for c in WeiboNote.__table__.columns}
|
||||
# 表结构层面无禁用列
|
||||
assert not (note_cols & FORBIDDEN_KEYS), \
|
||||
f"WeiboNote 表仍含禁用列: {note_cols & FORBIDDEN_KEYS}"
|
||||
# 行数据层面:creator_hash 正确、昵称脱敏、正文保留
|
||||
assert row.creator_hash and row.creator_hash != str(RAW_USER_ID)
|
||||
assert row.nickname != RAW_NICKNAME and "*" in row.nickname
|
||||
assert row.note_id == NOTE_ID
|
||||
assert "今天天气不错" in row.content
|
||||
# 确认没有 desc 列存任何用户描述
|
||||
assert "desc" not in note_cols
|
||||
|
||||
# ---- 4. comment:同上。注意 store_comment 实现会把 comment_id/note_id/create_time
|
||||
# 转成 int 后再构造 ORM,此处忠实复刻该转换以走通 BigInteger 列写入 ----
|
||||
cc = dict(captured_comment)
|
||||
cc["comment_id"] = int(cc["comment_id"])
|
||||
cc["note_id"] = int(cc.get("note_id", 0) or 0)
|
||||
cc["create_time"] = int(cc.get("create_time", 0) or 0)
|
||||
comment_obj = WeiboNoteComment(**cc) # 不抛异常 => key 与 ORM 列对得上
|
||||
session.add(comment_obj)
|
||||
session.commit()
|
||||
|
||||
crow = session.query(WeiboNoteComment).one()
|
||||
comment_cols = {c.name for c in WeiboNoteComment.__table__.columns}
|
||||
assert not (comment_cols & FORBIDDEN_KEYS), \
|
||||
f"WeiboNoteComment 表仍含禁用列: {comment_cols & FORBIDDEN_KEYS}"
|
||||
assert crow.creator_hash and crow.creator_hash != str(RAW_COMMENT_USER_ID)
|
||||
assert crow.nickname != RAW_COMMENT_NICKNAME and "*" in crow.nickname
|
||||
assert crow.comment_id == COMMENT_ID
|
||||
assert crow.note_id == NOTE_ID
|
||||
assert "说得好" in crow.content
|
||||
finally:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user