MediaCrawler/tests/test_douyin_no_user_info.py

# -*- coding: utf-8 -*-
"""
教学版回归测试(抖音 douyin):确保 store/douyin 存储链路不再持久化可定位真人的
用户个人信息。

教学版约定:用户 ID(uid/sec_uid/short_user_id/user_unique_id)/IP/头像/签名/性别
一律不持久化;原始 uid 经 tools.user_hash.anonymize_user_id 转成 creator_hash
(sha256 截断 16 位)写入;昵称保留但经 mask_nickname 中间脱敏。desc(作品描述)是
内容字段,保留。

覆盖:
1. test_douyin_aweme_masks_user_info
   - mock aweme_item(含 author.uid/sec_uid/short_id/unique_id/nickname/
     avatar_thumb.url_list/signature、aweme_item.ip_label、statistics、video/
     music/images 等内容字段)
   - FakeStore 捕获 update_douyin_aweme 产出的存储 dict
   - 断言:不含禁用键;含 creator_hash(≠原 uid);nickname 脱敏≠原文;
     原始敏感值不在任何存储值中;内容字段正常提取
2. test_douyin_comment_masks_user_info
   - 同上,针对 update_dy_aweme_comment(含嵌套 user.* / comment_item.ip_label /
     cid / text / image_list 等)
3. test_douyin_store_end_to_end_sqlite
   - FakeStore 捕获真实 dict → DouyinAweme(**captured) 构造(ORM 列校验:
     dict 多了已删列会 TypeError)
   - 端到端:monkeypatch db_session 为内存 SQLite engine + create_all,
     走真实 DouyinDbStoreImplement.store_content(含 int(aweme_id) / if title
     判断 / DouyinAweme(**content_item)),查询回读,验证 dict key 与删列后 ORM
     完全对得上
"""
import asyncio

import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import StaticPool

import config
import store.douyin as ds
from database import db_session
from database.models import Base, DouyinAweme, DouyinAwemeComment
from tools.user_hash import anonymize_user_id, mask_nickname


# 抖音教学版禁用字段(键):不得作为存储 dict 的 key 出现。
FORBIDDEN_KEYS = {
    "user_id", "sec_uid", "short_user_id", "user_unique_id",
    "avatar", "user_signature", "ip_location",
}


# ----------------------------- mock payload -----------------------------

def _build_aweme_item() -> dict:
    """贴近真实抖音结构的 mock aweme_item。
    含会被拍平的用户字段(uid/sec_uid/short_id/unique_id/avatar/signature)与
    ip_label,这些必须不落库;同时含 statistics/video/music/images 等内容字段,
    让 _extract_* 辅助函数返回非空值。"""
    return {
        "aweme_id": "7234567890123456",
        "aweme_type": 0,
        "desc": "这是一个作品描述,同时作为 title",
        "create_time": 1700000000,
        "ip_label": "上海",  # IP 归属地,禁用
        "author": {
            "uid": "9876543210",
            "sec_uid": "MS4wLjABAAAASecretSecUidForTest",
            "short_id": "88877766",
            "unique_id": "creator_unique_abc",
            "nickname": "抖音创作者",
            "avatar_thumb": {"url_list": ["http://x/avatar_thumb.jpg"]},
            "avatar_medium": {"url_list": ["http://x/avatar_medium.jpg"]},
            "signature": "这是创作者个人签名内容",
        },
        "statistics": {
            "digg_count": 100,
            "collect_count": 5,
            "comment_count": 20,
            "share_count": 3,
        },
        "video": {
            "raw_cover": {"url_list": ["", "http://x/cover.jpg"]},
            "origin_cover": {"url_list": ["", "http://x/cover2.jpg"]},
            "play_addr_h264": {"url_list": ["", "http://x/video_h264.mp4"]},
            "play_addr_256": {"url_list": ["", "http://x/video_256.mp4"]},
            "play_addr": {"url_list": ["", "http://x/video.mp4"]},
        },
        "music": {
            "play_url": {"uri": "http://x/music.mp3"},
        },
        "images": [
            {"url_list": ["", "http://x/note_img1.jpg"]},
            {"url_list": ["", "http://x/note_img2.jpg"]},
        ],
    }


def _build_comment_item() -> dict:
    """贴近真实抖音结构的 mock comment_item。aweme_id 须与传入
    update_dy_aweme_comment 的 aweme_id 一致。"""
    return {
        "aweme_id": "7234567890123456",
        "cid": "1111111111",
        "text": "这是一条评论内容",
        "create_time": 1700000001,
        "reply_id": "0",
        "reply_comment_total": 2,
        "digg_count": 5,
        "ip_label": "北京",  # IP 归属地,禁用
        "user": {
            "uid": "555666777",
            "sec_uid": "MS4wLjABBBBCommentSecUid",
            "short_id": "22233344",
            "unique_id": "commenter_unique_xyz",
            "nickname": "评论员小王",
            "avatar_medium": {"url_list": ["http://x/cavatar.jpg"]},
            "signature": "评论员个人签名内容",
        },
        "image_list": [
            {"origin_url": {"url_list": ["", "http://x/cimg.jpg"]}},
        ],
    }


# ----------------------------- 辅助 -----------------------------

class _FakeStore:
    """捕获存储 dict,不真正落库。"""

    def __init__(self):
        self.contents = []
        self.comments = []

    async def store_content(self, content_item):
        self.contents.append(dict(content_item))

    async def store_comment(self, comment_item):
        self.comments.append(dict(comment_item))


def _patch_factory(fake):
    """把 DouyinStoreFactory.create_store 替换为返回 fake,返回原方法用于还原。"""
    orig = ds.DouyinStoreFactory.create_store
    ds.DouyinStoreFactory.create_store = staticmethod(lambda: fake)
    return orig


def _assert_no_forbidden(captured: dict, label: str):
    hit = set(captured.keys()) & FORBIDDEN_KEYS
    assert not hit, f"[{label}] 存储 dict 仍含禁用键: {hit}"


def _assert_raw_values_absent(captured: dict, raw_values, label: str):
    """禁用的原始敏感值(uid/sec_uid/头像/签名/IP 等)不得出现在任何存储值里。
    creator_hash 是由 uid 派生的匿名哈希(已单独断言 ≠ 原文),不参与子串扫描。"""
    leaked = []
    for rv in raw_values:
        if not rv:
            continue
        for k, v in captured.items():
            if k == "creator_hash":
                continue
            if isinstance(v, str) and rv in v:
                leaked.append((k, rv))
    assert not leaked, f"[{label}] 存储 dict 中泄漏了原始敏感值: {leaked}"


# ----------------------------- 测试 -----------------------------

def test_douyin_aweme_masks_user_info():
    aweme = _build_aweme_item()
    raw_uid = aweme["author"]["uid"]
    raw_nick = aweme["author"]["nickname"]
    raw_sensitive = [
        raw_uid,
        aweme["author"]["sec_uid"],
        aweme["author"]["short_id"],
        aweme["author"]["unique_id"],
        aweme["author"]["avatar_thumb"]["url_list"][0],
        aweme["author"]["signature"],
        aweme["ip_label"],
    ]

    fake = _FakeStore()
    orig = _patch_factory(fake)
    try:
        asyncio.run(ds.update_douyin_aweme(aweme))
    finally:
        ds.DouyinStoreFactory.create_store = orig

    assert len(fake.contents) == 1
    captured = fake.contents[0]

    # 1. 禁用键不出现
    _assert_no_forbidden(captured, "douyin_aweme")
    # 2. 原始敏感值不泄漏到任何存储值
    _assert_raw_values_absent(captured, raw_sensitive, "douyin_aweme")
    # 3. creator_hash 存在、≠原 uid、与 anonymize_user_id 一致
    assert captured.get("creator_hash")
    assert captured["creator_hash"] != raw_uid
    assert captured["creator_hash"] == anonymize_user_id(raw_uid)
    # 4. nickname 保留但脱敏,≠原文,与 mask_nickname 一致
    assert captured.get("nickname")
    assert captured["nickname"] != raw_nick
    assert captured["nickname"] == mask_nickname(raw_nick)
    # 5. 内容字段保留(desc/title 是作品描述,不禁用)
    assert captured.get("desc") == aweme["desc"]
    assert captured.get("title") == aweme["desc"]
    # 6. _extract_* 内容字段正常提取(非空)
    assert captured.get("cover_url") == "http://x/cover.jpg"
    assert captured.get("video_download_url") == "http://x/video_h264.mp4"
    assert captured.get("music_download_url") == "http://x/music.mp3"
    assert "http://x/note_img1.jpg" in captured.get("note_download_url", "")
    # 7. 互动数据拍平
    assert captured.get("liked_count") == "100"
    assert captured.get("collected_count") == "5"
    assert captured.get("comment_count") == "20"
    assert captured.get("share_count") == "3"
    assert captured.get("aweme_url") == f"https://www.douyin.com/video/{aweme['aweme_id']}"


def test_douyin_comment_masks_user_info():
    aweme_id = "7234567890123456"
    comment = _build_comment_item()
    raw_uid = comment["user"]["uid"]
    raw_nick = comment["user"]["nickname"]
    raw_sensitive = [
        raw_uid,
        comment["user"]["sec_uid"],
        comment["user"]["short_id"],
        comment["user"]["unique_id"],
        comment["user"]["avatar_medium"]["url_list"][0],
        comment["user"]["signature"],
        comment["ip_label"],
    ]

    fake = _FakeStore()
    orig = _patch_factory(fake)
    try:
        asyncio.run(ds.update_dy_aweme_comment(aweme_id, comment))
    finally:
        ds.DouyinStoreFactory.create_store = orig

    assert len(fake.comments) == 1
    captured = fake.comments[0]

    _assert_no_forbidden(captured, "douyin_comment")
    _assert_raw_values_absent(captured, raw_sensitive, "douyin_comment")

    assert captured.get("creator_hash")
    assert captured["creator_hash"] != raw_uid
    assert captured["creator_hash"] == anonymize_user_id(raw_uid)
    assert captured.get("nickname")
    assert captured["nickname"] != raw_nick
    assert captured["nickname"] == mask_nickname(raw_nick)

    # 评论内容/ID 保留
    assert captured.get("content") == comment["text"]
    assert captured.get("comment_id") == comment["cid"]
    assert captured.get("aweme_id") == aweme_id
    assert captured.get("parent_comment_id") == "0"
    assert captured.get("sub_comment_count") == "2"
    # 评论图片提取
    assert captured.get("pictures") == "http://x/cimg.jpg"


def test_douyin_store_end_to_end_sqlite(monkeypatch):
    aweme = _build_aweme_item()
    raw_uid = aweme["author"]["uid"]
    raw_nick = aweme["author"]["nickname"]

    # ---- 1. FakeStore 捕获 update_douyin_aweme 产出的真实 dict ----
    fake = _FakeStore()
    orig = _patch_factory(fake)
    try:
        asyncio.run(ds.update_douyin_aweme(aweme))
    finally:
        ds.DouyinStoreFactory.create_store = orig
    assert len(fake.contents) == 1
    captured = fake.contents[0]

    # ---- 2. DouyinAweme(**captured) 构造校验 ----
    # dict 多了已删列会 TypeError,少了非空必填列 SQLAlchemy 也会报错;
    # 此处证明 captured 的 key 与删列后 ORM 列完全对得上,不抛异常。
    obj = DouyinAweme(**captured)
    assert obj.aweme_id == aweme["aweme_id"]
    assert obj.creator_hash == anonymize_user_id(raw_uid)
    assert obj.creator_hash != raw_uid
    assert obj.nickname == mask_nickname(raw_nick)
    assert obj.nickname != raw_nick
    assert obj.title == aweme["desc"]
    assert obj.desc == aweme["desc"]
    _assert_no_forbidden(captured, "douyin_aweme_orm_construct")

    # ---- 3. 端到端:内存 SQLite + 真实 store_content ----
    # 用 StaticPool 保证 :memory: 库在同一个连接上持久(跨 session 可见)。
    engine = create_async_engine(
        "sqlite+aiosqlite:///:memory:",
        poolclass=StaticPool,
    )
    # 让 db_session 用内存 engine;SAVE_DATA_OPTION=db 让工厂走 DouyinDbStoreImplement
    monkeypatch.setattr(db_session, "get_async_engine", lambda *a, **kw: engine)
    monkeypatch.setattr(config, "SAVE_DATA_OPTION", "db")

    async def _scenario():
        # 建表
        async with engine.begin() as conn:
            await conn.run_sync(Base.metadata.create_all)
        # 真实 store_content 路径(含 int(aweme_id) 与 if content_item.get("title") 判断)
        await ds.update_douyin_aweme(aweme)
        # 查询回读
        async with db_session.get_session() as session:
            res = await session.execute(
                select(DouyinAweme).where(DouyinAweme.aweme_id == aweme["aweme_id"])
            )
            row = res.scalar_one_or_none()
        await engine.dispose()
        return row

    row = asyncio.run(_scenario())

    # ---- 4. 回读断言 ----
    assert row is not None, "作品未写入 SQLite"
    assert row.aweme_id == aweme["aweme_id"]
    assert row.creator_hash == anonymize_user_id(raw_uid)
    assert row.creator_hash != raw_uid
    assert row.nickname == mask_nickname(raw_nick)
    assert row.nickname != raw_nick
    assert row.desc == aweme["desc"]
    assert row.title == aweme["desc"]
    assert row.cover_url == "http://x/cover.jpg"
    assert row.video_download_url == "http://x/video_h264.mp4"
    assert row.music_download_url == "http://x/music.mp3"
    # 禁用列在 ORM 上不存在(自省)
    orm_cols = {c.name for c in DouyinAweme.__table__.columns}
    assert not (orm_cols & FORBIDDEN_KEYS), f"ORM 仍含禁用列: {orm_cols & FORBIDDEN_KEYS}"
    # captured 的所有 key 都是合法 ORM 列(无悬空 key)
    assert set(captured.keys()).issubset(orm_cols), (
        f"captured 含非 ORM 列: {set(captured.keys()) - orm_cols}"
    )

    # 评论同样做一次 ORM 构造校验(证明 comment dict key 对得上)
    comment = _build_comment_item()
    fake_c = _FakeStore()
    orig_c = _patch_factory(fake_c)
    try:
        asyncio.run(ds.update_dy_aweme_comment(comment["aweme_id"], comment))
    finally:
        ds.DouyinStoreFactory.create_store = orig_c
    captured_comment = fake_c.comments[0]
    comment_obj = DouyinAwemeComment(**captured_comment)  # 不抛异常即对得上
    assert comment_obj.comment_id == comment["cid"]
    assert comment_obj.creator_hash == anonymize_user_id(comment["user"]["uid"])
    assert comment_obj.nickname == mask_nickname(comment["user"]["nickname"])
    _assert_no_forbidden(captured_comment, "douyin_comment_orm_construct")
    comment_orm_cols = {c.name for c in DouyinAwemeComment.__table__.columns}
    assert set(captured_comment.keys()).issubset(comment_orm_cols), (
        f"captured_comment 含非 ORM 列: {set(captured_comment.keys()) - comment_orm_cols}"
    )


if __name__ == "__main__":
    pytest.main([__file__, "-v"])