From 9f4f8bf76880106f5ecbf0c56b3e3dcbb56bc741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Wed, 1 Jul 2026 13:09:55 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E6=95=99=E5=AD=A6=E7=89=88?= =?UTF-8?q?=E7=A7=BB=E9=99=A4=E5=85=A8=E5=B9=B3=E5=8F=B0=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E4=B8=AA=E4=BA=BA=E4=BF=A1=E6=81=AF=E9=87=87=E9=9B=86=E4=B8=8E?= =?UTF-8?q?=E6=8C=81=E4=B9=85=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 用户 ID 转为匿名 creator_hash，昵称中间脱敏，IP/头像/主页/签名/性别不再采集 - 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台 - 删除 7 张 creator 档案 ORM 表，15 张内容/评论表新增 creator_hash 列 - B 站禁用粉丝/关注/联系人列表抓取 - 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试 测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed) --- database/models.py | 225 +++-------------- media_platform/bilibili/core.py | 8 +- media_platform/tieba/help.py | 84 +++---- media_platform/zhihu/client.py | 20 +- media_platform/zhihu/core.py | 7 +- media_platform/zhihu/help.py | 37 +-- model/m_baidu_tieba.py | 22 +- model/m_zhihu.py | 29 +-- store/bilibili/__init__.py | 74 ++---- store/bilibili/_store_impl.py | 76 +----- store/douyin/__init__.py | 42 +--- store/douyin/_store_impl.py | 39 +-- store/kuaishou/__init__.py | 34 +-- store/kuaishou/_store_impl.py | 17 +- store/tieba/__init__.py | 6 +- store/tieba/_store_impl.py | 37 +-- store/weibo/__init__.py | 49 ++-- store/weibo/_store_impl.py | 53 ++-- store/xhs/__init__.py | 52 +--- store/xhs/_store_impl.py | 72 +----- store/zhihu/_store_impl.py | 66 +---- tests/test_douyin_no_user_info.py | 362 ++++++++++++++++++++++++++++ tests/test_kuaishou_no_user_info.py | 284 ++++++++++++++++++++++ tests/test_no_user_info.py | 239 ++++++++++++++++++ tests/test_weibo_no_user_info.py | 278 +++++++++++++++++++++ tools/user_hash.py | 36 +++ 26 files changed, 1426 insertions(+), 822 deletions(-) create mode 100644 tests/test_douyin_no_user_info.py create mode 100644 tests/test_kuaishou_no_user_info.py create mode 100644 tests/test_no_user_info.py create mode 100644 tests/test_weibo_no_user_info.py create mode 100644 tools/user_hash.py diff --git a/database/models.py b/database/models.py index ac8d1cd..011777d 100644 --- a/database/models.py +++ b/database/models.py @@ -15,6 +15,14 @@ # # 详细许可条款请参阅项目根目录下的LICENSE文件。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 +# +# 教学版说明：为防止爬取到的用户个人信息被用于定位真人并私信骚扰， +# 本 ORM 不再持久化任何可识别用户的字段（用户 ID、IP 归属地、头像、 +# 主页链接、签名、性别等一律不落库）。原始用户 ID 在提取层经 +# tools.user_hash.anonymize_user_id 转为匿名 creator_hash 后写入， +# 仅用于"同一创作者"的内容分组；昵称保留但经 mask_nickname 中间脱敏。 +# 创作者个人档案表（XhsCreator/DyCreator/WeiboCreator/TiebaCreator/ +# ZhihuCreator/BilibiliUpInfo/BilibiliContactInfo）已整体移除。 from sqlalchemy import create_engine, Column, Integer, Text, String, BigInteger from sqlalchemy.ext.declarative import declarative_base @@ -27,9 +35,8 @@ class BilibiliVideo(Base): id = Column(Integer, primary_key=True, comment='主键ID') video_id = Column(BigInteger, nullable=False, index=True, unique=True, comment='视频ID') video_url = Column(Text, nullable=False, comment='视频URL') - user_id = Column(BigInteger, index=True, comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') liked_count = Column(Integer, comment='点赞数') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') @@ -50,11 +57,8 @@ class BilibiliVideo(Base): class BilibiliVideoComment(Base): __tablename__ = 'bilibili_video_comment' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - sex = Column(Text, comment='性别') - sign = Column(Text, comment='签名') - avatar = Column(Text, comment='头像') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') comment_id = Column(BigInteger, index=True, comment='评论ID') @@ -65,41 +69,12 @@ class BilibiliVideoComment(Base): parent_comment_id = Column(String(255), comment='父评论ID') like_count = Column(Text, default='0', comment='点赞数') -class BilibiliUpInfo(Base): - __tablename__ = 'bilibili_up_info' - id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(BigInteger, index=True, comment='用户ID') - nickname = Column(Text, comment='用户昵称') - sex = Column(Text, comment='性别') - sign = Column(Text, comment='签名') - avatar = Column(Text, comment='头像') - add_ts = Column(BigInteger, comment='添加时间戳') - last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - total_fans = Column(Integer, comment='总粉丝数') - total_liked = Column(Integer, comment='总获赞数') - user_rank = Column(Integer, comment='用户等级') - is_official = Column(Integer, comment='是否官方认证') - -class BilibiliContactInfo(Base): - __tablename__ = 'bilibili_contact_info' - id = Column(Integer, primary_key=True, comment='主键ID') - up_id = Column(BigInteger, index=True, comment='UP主ID') - fan_id = Column(BigInteger, index=True, comment='粉丝ID') - up_name = Column(Text, comment='UP主名称') - fan_name = Column(Text, comment='粉丝名称') - up_sign = Column(Text, comment='UP主签名') - fan_sign = Column(Text, comment='粉丝签名') - up_avatar = Column(Text, comment='UP主头像') - fan_avatar = Column(Text, comment='粉丝头像') - add_ts = Column(BigInteger, comment='添加时间戳') - last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - class BilibiliUpDynamic(Base): __tablename__ = 'bilibili_up_dynamic' id = Column(Integer, primary_key=True, comment='主键ID') dynamic_id = Column(BigInteger, index=True, comment='动态ID') - user_id = Column(String(255), comment='用户ID') - user_name = Column(Text, comment='用户名称') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + user_name = Column(Text, comment='用户名称(已脱敏)') text = Column(Text, comment='动态内容') type = Column(Text, comment='动态类型') pub_ts = Column(BigInteger, comment='发布时间戳') @@ -112,14 +87,8 @@ class BilibiliUpDynamic(Base): class DouyinAweme(Base): __tablename__ = 'douyin_aweme' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - sec_uid = Column(String(255), comment='安全用户ID') - short_user_id = Column(String(255), comment='短用户ID') - user_unique_id = Column(String(255), comment='用户唯一ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - user_signature = Column(Text, comment='用户签名') - ip_location = Column(Text, comment='IP地址位置') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') aweme_id = Column(BigInteger, index=True, comment='作品ID') @@ -141,14 +110,8 @@ class DouyinAweme(Base): class DouyinAwemeComment(Base): __tablename__ = 'douyin_aweme_comment' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - sec_uid = Column(String(255), comment='安全用户ID') - short_user_id = Column(String(255), comment='短用户ID') - user_unique_id = Column(String(255), comment='用户唯一ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - user_signature = Column(Text, comment='用户签名') - ip_location = Column(Text, comment='IP地址位置') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') comment_id = Column(BigInteger, index=True, comment='评论ID') @@ -160,28 +123,11 @@ class DouyinAwemeComment(Base): like_count = Column(Text, default='0', comment='点赞数') pictures = Column(Text, default='', comment='图片') -class DyCreator(Base): - __tablename__ = 'dy_creator' - id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - ip_location = Column(Text, comment='IP地址位置') - add_ts = Column(BigInteger, comment='添加时间戳') - last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - desc = Column(Text, comment='描述') - gender = Column(Text, comment='性别') - follows = Column(Text, comment='关注数') - fans = Column(Text, comment='粉丝数') - interaction = Column(Text, comment='互动数') - videos_count = Column(String(255), comment='视频数量') - class KuaishouVideo(Base): __tablename__ = 'kuaishou_video' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(64), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') video_id = Column(String(255), index=True, comment='视频ID') @@ -199,9 +145,8 @@ class KuaishouVideo(Base): class KuaishouVideoComment(Base): __tablename__ = 'kuaishou_video_comment' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(Text, comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') comment_id = Column(BigInteger, index=True, comment='评论ID') @@ -213,12 +158,8 @@ class KuaishouVideoComment(Base): class WeiboNote(Base): __tablename__ = 'weibo_note' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - gender = Column(Text, comment='性别') - profile_url = Column(Text, comment='个人主页URL') - ip_location = Column(Text, default='', comment='IP地址位置') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') note_id = Column(BigInteger, index=True, comment='笔记ID') @@ -234,12 +175,8 @@ class WeiboNote(Base): class WeiboNoteComment(Base): __tablename__ = 'weibo_note_comment' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - gender = Column(Text, comment='性别') - profile_url = Column(Text, comment='个人主页URL') - ip_location = Column(Text, default='', comment='IP地址位置') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') comment_id = Column(BigInteger, index=True, comment='评论ID') @@ -251,44 +188,11 @@ class WeiboNoteComment(Base): sub_comment_count = Column(Text, comment='子评论数') parent_comment_id = Column(String(255), comment='父评论ID') -class WeiboCreator(Base): - __tablename__ = 'weibo_creator' - id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - ip_location = Column(Text, comment='IP地址位置') - add_ts = Column(BigInteger, comment='添加时间戳') - last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - desc = Column(Text, comment='描述') - gender = Column(Text, comment='性别') - follows = Column(Text, comment='关注数') - fans = Column(Text, comment='粉丝数') - tag_list = Column(Text, comment='标签列表') - -class XhsCreator(Base): - __tablename__ = 'xhs_creator' - id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - ip_location = Column(Text, comment='IP地址位置') - add_ts = Column(BigInteger, comment='添加时间戳') - last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - desc = Column(Text, comment='描述') - gender = Column(Text, comment='性别') - follows = Column(Text, comment='关注数') - fans = Column(Text, comment='粉丝数') - interaction = Column(Text, comment='互动数') - tag_list = Column(Text, comment='标签列表') - class XhsNote(Base): __tablename__ = 'xhs_note' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - ip_location = Column(Text, comment='IP地址位置') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') note_id = Column(String(255), index=True, comment='笔记ID') @@ -311,10 +215,8 @@ class XhsNote(Base): class XhsNoteComment(Base): __tablename__ = 'xhs_note_comment' id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(255), comment='用户ID') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - ip_location = Column(Text, comment='IP地址位置') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') comment_id = Column(String(255), index=True, comment='评论ID') @@ -334,15 +236,13 @@ class TiebaNote(Base): desc = Column(Text, comment='笔记描述') note_url = Column(Text, comment='笔记URL') publish_time = Column(String(255), index=True, comment='发布时间') - user_link = Column(Text, default='', comment='用户链接') - user_nickname = Column(Text, default='', comment='用户昵称') - user_avatar = Column(Text, default='', comment='用户头像') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + user_nickname = Column(Text, default='', comment='用户昵称(已脱敏)') tieba_id = Column(String(255), default='', comment='贴吧ID') tieba_name = Column(Text, comment='贴吧名称') tieba_link = Column(Text, comment='贴吧链接') total_replay_num = Column(Integer, default=0, comment='总回复数') total_replay_page = Column(Integer, default=0, comment='总回复页数') - ip_location = Column(Text, default='', comment='IP地址位置') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') source_keyword = Column(Text, default='', comment='来源关键词') @@ -353,35 +253,18 @@ class TiebaComment(Base): comment_id = Column(String(255), index=True, comment='评论ID') parent_comment_id = Column(String(255), default='', comment='父评论ID') content = Column(Text, comment='评论内容') - user_link = Column(Text, default='', comment='用户链接') - user_nickname = Column(Text, default='', comment='用户昵称') - user_avatar = Column(Text, default='', comment='用户头像') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + user_nickname = Column(Text, default='', comment='用户昵称(已脱敏)') tieba_id = Column(String(255), default='', comment='贴吧ID') tieba_name = Column(Text, comment='贴吧名称') tieba_link = Column(Text, comment='贴吧链接') publish_time = Column(String(255), index=True, comment='发布时间') - ip_location = Column(Text, default='', comment='IP地址位置') sub_comment_count = Column(Integer, default=0, comment='子评论数') note_id = Column(String(255), index=True, comment='笔记ID') note_url = Column(Text, comment='笔记URL') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') -class TiebaCreator(Base): - __tablename__ = 'tieba_creator' - id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(64), comment='用户ID') - user_name = Column(Text, comment='用户名') - nickname = Column(Text, comment='用户昵称') - avatar = Column(Text, comment='用户头像') - ip_location = Column(Text, comment='IP地址位置') - add_ts = Column(BigInteger, comment='添加时间戳') - last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - gender = Column(Text, comment='性别') - follows = Column(Text, comment='关注数') - fans = Column(Text, comment='粉丝数') - registration_duration = Column(Text, comment='注册时长') - class ZhihuContent(Base): __tablename__ = 'zhihu_content' id = Column(Integer, primary_key=True, comment='主键ID') @@ -397,19 +280,11 @@ class ZhihuContent(Base): voteup_count = Column(Integer, default=0, comment='赞同数') comment_count = Column(Integer, default=0, comment='评论数') source_keyword = Column(Text, comment='来源关键词') - user_id = Column(String(255), comment='用户ID') - user_link = Column(Text, comment='用户链接') - user_nickname = Column(Text, comment='用户昵称') - user_avatar = Column(Text, comment='用户头像') - user_url_token = Column(Text, comment='用户URL Token') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + user_nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - # persist-1 - # Reason: Fixed ORM model definition error, ensuring consistency with database table structure. - # Side effects: None - # Rollback strategy: Restore this line - class ZhihuComment(Base): __tablename__ = 'zhihu_comment' id = Column(Integer, primary_key=True, comment='主键ID') @@ -417,36 +292,12 @@ class ZhihuComment(Base): parent_comment_id = Column(String(64), comment='父评论ID') content = Column(Text, comment='评论内容') publish_time = Column(String(32), index=True, comment='发布时间') - ip_location = Column(Text, comment='IP地址位置') sub_comment_count = Column(Integer, default=0, comment='子评论数') like_count = Column(Integer, default=0, comment='点赞数') dislike_count = Column(Integer, default=0, comment='点踩数') content_id = Column(String(64), index=True, comment='内容ID') content_type = Column(Text, comment='内容类型') - user_id = Column(String(64), comment='用户ID') - user_link = Column(Text, comment='用户链接') - user_nickname = Column(Text, comment='用户昵称') - user_avatar = Column(Text, comment='用户头像') - add_ts = Column(BigInteger, comment='添加时间戳') - last_modify_ts = Column(BigInteger, comment='最后修改时间戳') - -class ZhihuCreator(Base): - __tablename__ = 'zhihu_creator' - id = Column(Integer, primary_key=True, comment='主键ID') - user_id = Column(String(64), unique=True, index=True, comment='用户ID') - user_link = Column(Text, comment='用户链接') - user_nickname = Column(Text, comment='用户昵称') - user_avatar = Column(Text, comment='用户头像') - url_token = Column(Text, comment='URL Token') - gender = Column(Text, comment='性别') - ip_location = Column(Text, comment='IP地址位置') - follows = Column(Integer, default=0, comment='关注数') - fans = Column(Integer, default=0, comment='粉丝数') - anwser_count = Column(Integer, default=0, comment='回答数') - video_count = Column(Integer, default=0, comment='视频数') - question_count = Column(Integer, default=0, comment='问题数') - article_count = Column(Integer, default=0, comment='文章数') - column_count = Column(Integer, default=0, comment='专栏数') - get_voteup_count = Column(Integer, default=0, comment='获赞数') + creator_hash = Column(String(64), index=True, comment='创作者匿名哈希') + user_nickname = Column(Text, comment='用户昵称(已脱敏)') add_ts = Column(BigInteger, comment='添加时间戳') last_modify_ts = Column(BigInteger, comment='最后修改时间戳') diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index c051fd6..14b5099 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -646,14 +646,14 @@ class BilibiliCrawler(AbstractCrawler): """ async with semaphore: creator_unhandled_info: Dict = await self.bili_client.get_creator_info(creator_id) + # 教学版：仅保留动态所需的最少字段(内存临时用)，不持久化创作者个人资料。 creator_info: Dict = { "id": creator_id, "name": creator_unhandled_info.get("name"), - "sign": creator_unhandled_info.get("sign"), - "avatar": creator_unhandled_info.get("face"), } - await self.get_fans(creator_info, semaphore) - await self.get_followings(creator_info, semaphore) + # 教学版：不再爬取粉丝/关注列表(其他用户的个人信息)，防骚扰。 + # await self.get_fans(creator_info, semaphore) + # await self.get_followings(creator_info, semaphore) await self.get_dynamics(creator_info, semaphore) async def get_fans(self, creator_info: Dict, semaphore: asyncio.Semaphore): diff --git a/media_platform/tieba/help.py b/media_platform/tieba/help.py index 89c1f5b..42b36e1 100644 --- a/media_platform/tieba/help.py +++ b/media_platform/tieba/help.py @@ -30,6 +30,7 @@ from parsel import Selector from constant import baidu_tieba as const from model.m_baidu_tieba import TiebaComment, TiebaCreator, TiebaNote from tools import utils +from tools.user_hash import anonymize_user_id, mask_nickname GENDER_MALE = "sex_male" GENDER_FEMALE = "sex_female" @@ -143,9 +144,8 @@ class TieBaExtractor: publish_time=utils.get_time_str_from_unix_time( item.get("time") or item.get("create_time") or 0 ), - user_link="", - user_nickname=user.get("show_nickname") or user.get("user_name") or "", - user_avatar=user.get("portrait") or user.get("portraith") or "", + creator_hash=anonymize_user_id(user.get("id") or user.get("portrait") or ""), + user_nickname=mask_nickname(user.get("show_nickname") or user.get("user_name") or ""), tieba_name=tieba_name, tieba_link=self._tieba_link_from_name(tieba_name), total_replay_num=item.get("post_num") or 0, @@ -177,14 +177,12 @@ class TieBaExtractor: publish_time=utils.get_time_str_from_unix_time( first_floor.get("time") or thread.get("create_time") or 0 ), - user_link=self._api_user_link(author), - user_nickname=author.get("name_show") or author.get("name") or "", - user_avatar=self._api_user_avatar(author), + creator_hash=anonymize_user_id(self._api_user_link(author)), + user_nickname=mask_nickname(author.get("name_show") or author.get("name") or ""), tieba_name=tieba_name, tieba_link=self._tieba_link_from_name(tieba_name), total_replay_num=thread.get("reply_num") or 0, total_replay_page=page.get("total_page") or 0, - ip_location=author.get("ip_address") or "", ) return note @@ -210,13 +208,11 @@ class TieBaExtractor: sub_comment_count=item.get("sub_post_number") or 0, content=self._extract_api_content_text(item.get("content")), note_url=note_detail.note_url, - user_link=self._api_user_link(user), - user_nickname=user.get("name_show") or user.get("name") or "", - user_avatar=self._api_user_avatar(user), + creator_hash=anonymize_user_id(self._api_user_link(user)), + user_nickname=mask_nickname(user.get("name_show") or user.get("name") or ""), tieba_id=tieba_id, tieba_name=tieba_name, tieba_link=tieba_link, - ip_location=user.get("ip_address") or "", publish_time=utils.get_time_str_from_unix_time(item.get("time") or 0), note_id=note_detail.note_id, ) @@ -230,20 +226,11 @@ class TieBaExtractor: user = api_data.get("data", {}).get("user", {}) if not user: raise ValueError(f"Creator API response does not contain user info: {api_data}") - gender_value = user.get("sex", user.get("gender", 0)) - gender = "Unknown" - if gender_value == 1: - gender = "Male" - elif gender_value == 2: - gender = "Female" + # 教学版：创作者个人资料不再落库，仅保留匿名哈希与脱敏昵称作内存对象。 return TiebaCreator( - user_id=str(user.get("id", "")), - user_name=str(user.get("name", "")), - nickname=str(user.get("name_show") or user.get("name") or ""), - avatar=self._api_user_avatar(user), - gender=gender, - ip_location=str(user.get("ip_address", "")), + creator_hash=anonymize_user_id(str(user.get("id", ""))), + user_nickname=mask_nickname(str(user.get("name_show") or user.get("name") or "")), follows=int(user.get("concern_num") or 0), fans=int(user.get("fans_num") or 0), registration_duration=str(user.get("tb_age", "")), @@ -370,10 +357,10 @@ class TieBaExtractor: post, f".//div[{extractor._class_contains('p_content')}]" ), note_url=note_url, - user_nickname=extractor._selector_text( + creator_hash=anonymize_user_id(extractor._absolute_url(user_selector.xpath("./@href").get(default=""))), + user_nickname=mask_nickname(extractor._selector_text( post, ".//a[contains(@href, '/home/main')][1]" - ), - user_link=extractor._absolute_url(user_selector.xpath("./@href").get(default="")), + )), tieba_name=extractor._selector_text( post, f".//a[{extractor._class_contains('p_forum')}][1]" ), @@ -451,8 +438,8 @@ class TieBaExtractor: title=title, desc=desc, note_url=f"{const.TIEBA_URL}/p/{note_id}", - user_nickname=user_nickname, - user_link="", + creator_hash="", + user_nickname=mask_nickname(user_nickname), tieba_name=tieba_name, tieba_link=tieba_link, publish_time=publish_time, @@ -499,8 +486,8 @@ class TieBaExtractor: post_selector, f".//div[{self._class_contains('threadlist_abs')}]" ), note_url=const.TIEBA_URL + f"/p/{note_id}", - user_link=self._absolute_url(user_selector.xpath("./@href").get(default="")), - user_nickname=user_nickname, + creator_hash=anonymize_user_id(self._absolute_url(user_selector.xpath("./@href").get(default=""))), + user_nickname=mask_nickname(user_nickname), tieba_name=tieba_name, tieba_link=tieba_link, total_replay_num=post_field_value.get("reply_num", 0), @@ -548,18 +535,14 @@ class TieBaExtractor: title=content_selector.xpath("//title/text()").get(default="").strip(), desc=content_selector.xpath("//meta[@name='description']/@content").get(default="").strip(), note_url=const.TIEBA_URL + f"/p/{note_id}", - user_link=self._absolute_url(author_link), - user_nickname=( + creator_hash=anonymize_user_id(self._absolute_url(author_link)), + user_nickname=mask_nickname( self._selector_text(first_floor_selector, f".//a[{self._class_contains('p_author_name')}][1]") or author_value.get("user_nickname") or author_value.get("user_name", "") ), - user_avatar=first_floor_selector.xpath( - f".//a[{self._class_contains('p_author_face')}]//img/@src" - ).get(default="").strip(), tieba_name=tieba_name, tieba_link=tieba_link, - ip_location=ip_location, publish_time=publish_time, total_replay_num=( thread_num_infos[0].xpath("./text()").get(default="0").strip() @@ -623,13 +606,11 @@ class TieBaExtractor: sub_comment_count=comment_content_value.get("comment_num") or 0, content=utils.extract_text_from_html(content_html), note_url=const.TIEBA_URL + f"/p/{note_id}", - user_link=self._absolute_url(user_selector.xpath("./@href").get(default="")), - user_nickname=user_nickname, - user_avatar=user_avatar, + creator_hash=anonymize_user_id(self._absolute_url(user_selector.xpath("./@href").get(default=""))), + user_nickname=mask_nickname(user_nickname), tieba_id=str(comment_content_value.get("forum_id", "")), tieba_name=tieba_name, tieba_link=tieba_link, - ip_location=ip_location, publish_time=publish_time, note_id=note_id, ) @@ -662,9 +643,8 @@ class TieBaExtractor: comment_ele.xpath(f".//span[{self._class_contains('lzl_content_main')}]").get(default="")) comment = TiebaComment( comment_id=str(comment_value.get("spid")), content=content, - user_link=self._absolute_url(comment_user_a_selector.xpath("./@href").get(default="")), - user_nickname=str(comment_value.get("showname") or ""), - user_avatar=comment_user_a_selector.xpath("./img/@src").get(default=""), + creator_hash=anonymize_user_id(self._absolute_url(comment_user_a_selector.xpath("./@href").get(default=""))), + user_nickname=mask_nickname(str(comment_value.get("showname") or "")), publish_time=self._selector_text(comment_ele, f".//span[{self._class_contains('lzl_time')}]"), parent_comment_id=parent_comment.comment_id, note_id=parent_comment.note_id, note_url=parent_comment.note_url, @@ -695,13 +675,13 @@ class TieBaExtractor: if len(follow_fans_selector) == 2: follows, fans = self.extract_follow_and_fans(follow_fans_selector) user_content = userinfo_userdata_selector.get(default='') - return TiebaCreator(user_id=user_id, user_name=user_name, - nickname=selector.xpath(".//span[@class='userinfo_username ']/text()").get( - default='').strip(), - avatar=selector.xpath(".//div[@class='userinfo_left_head']//img/@src").get( - default='').strip(), - gender=self.extract_gender(user_content), - ip_location=self.extract_ip(user_content), + # 教学版：创作者个人资料不再落库，仅保留匿名哈希与脱敏昵称作内存对象。 + return TiebaCreator(creator_hash=anonymize_user_id(user_id or user_link), + user_nickname=mask_nickname( + selector.xpath(".//span[@class='userinfo_username ']/text()").get( + default='').strip() + or user_name + ), follows=follows, fans=fans, registration_duration=self.extract_registration_duration(user_content) @@ -860,8 +840,8 @@ def test_extract_tieba_note_sub_comments(): with open("test_data/note_sub_comments.html", "r", encoding="utf-8") as f: content = f.read() extractor = TieBaExtractor() - fake_parment_comment = TiebaComment(comment_id="123456", content="content", user_link="user_link", - user_nickname="user_nickname", user_avatar="user_avatar", + fake_parment_comment = TiebaComment(comment_id="123456", content="content", creator_hash="creator_hash", + user_nickname="user_nickname", publish_time="publish_time", parent_comment_id="parent_comment_id", note_id="note_id", note_url="note_url", tieba_id="tieba_id", tieba_name="tieba_name", ) diff --git a/media_platform/zhihu/client.py b/media_platform/zhihu/client.py index 7e11991..073928f 100644 --- a/media_platform/zhihu/client.py +++ b/media_platform/zhihu/client.py @@ -459,11 +459,11 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin): } return await self.get(uri, params) - async def get_all_anwser_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]: + async def get_all_anwser_by_creator(self, url_token: str, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]: """ Get all answers by creator Args: - creator: Creator information + url_token: Creator url token (in-memory only, not persisted) crawl_interval: Crawl delay interval in seconds callback: Callback after completing one crawl @@ -475,10 +475,10 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin): offset: int = 0 limit: int = 20 while not is_end: - res = await self.get_creator_answers(creator.url_token, offset, limit) + res = await self.get_creator_answers(url_token, offset, limit) if not res: break - utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {creator.url_token} answers: {res}") + utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {url_token} answers: {res}") paging_info = res.get("paging", {}) is_end = paging_info.get("is_end") contents = self._extractor.extract_content_list_from_creator(res.get("data")) @@ -491,14 +491,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin): async def get_all_articles_by_creator( self, - creator: ZhihuCreator, + url_token: str, crawl_interval: float = 1.0, callback: Optional[Callable] = None, ) -> List[ZhihuContent]: """ Get all articles by creator Args: - creator: + url_token: Creator url token (in-memory only, not persisted) crawl_interval: callback: @@ -510,7 +510,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin): offset: int = 0 limit: int = 20 while not is_end: - res = await self.get_creator_articles(creator.url_token, offset, limit) + res = await self.get_creator_articles(url_token, offset, limit) if not res: break paging_info = res.get("paging", {}) @@ -525,14 +525,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin): async def get_all_videos_by_creator( self, - creator: ZhihuCreator, + url_token: str, crawl_interval: float = 1.0, callback: Optional[Callable] = None, ) -> List[ZhihuContent]: """ Get all videos by creator Args: - creator: + url_token: Creator url token (in-memory only, not persisted) crawl_interval: callback: @@ -544,7 +544,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin): offset: int = 0 limit: int = 20 while not is_end: - res = await self.get_creator_videos(creator.url_token, offset, limit) + res = await self.get_creator_videos(url_token, offset, limit) if not res: break paging_info = res.get("paging", {}) diff --git a/media_platform/zhihu/core.py b/media_platform/zhihu/core.py index a7ea9df..7b21e3b 100644 --- a/media_platform/zhihu/core.py +++ b/media_platform/zhihu/core.py @@ -276,27 +276,26 @@ class ZhihuCrawler(AbstractCrawler): utils.logger.info( f"[ZhihuCrawler.get_creators_and_notes] Creator info: {createor_info}" ) - await zhihu_store.save_creator(creator=createor_info) # By default, only answer information is extracted, uncomment below if articles and videos are needed # Get all anwser information of the creator all_content_list = await self.zhihu_client.get_all_anwser_by_creator( - creator=createor_info, + url_token=user_url_token, crawl_interval=config.CRAWLER_MAX_SLEEP_SEC, callback=zhihu_store.batch_update_zhihu_contents, ) # Get all articles of the creator's contents # all_content_list = await self.zhihu_client.get_all_articles_by_creator( - # creator=createor_info, + # url_token=user_url_token, # crawl_interval=config.CRAWLER_MAX_SLEEP_SEC, # callback=zhihu_store.batch_update_zhihu_contents # ) # Get all videos of the creator's contents # all_content_list = await self.zhihu_client.get_all_videos_by_creator( - # creator=createor_info, + # url_token=user_url_token, # crawl_interval=config.CRAWLER_MAX_SLEEP_SEC, # callback=zhihu_store.batch_update_zhihu_contents # ) diff --git a/media_platform/zhihu/help.py b/media_platform/zhihu/help.py index aa8338d..24877df 100644 --- a/media_platform/zhihu/help.py +++ b/media_platform/zhihu/help.py @@ -30,6 +30,7 @@ from constant import zhihu as zhihu_constant from model.m_zhihu import ZhihuComment, ZhihuContent, ZhihuCreator from tools import utils from tools.crawler_util import extract_text_from_html +from tools.user_hash import anonymize_user_id, mask_nickname ZHIHU_SGIN_JS = None @@ -120,11 +121,8 @@ class ZhihuExtractor: # extract author info author_info = self._extract_content_or_comment_author(answer.get("author")) - res.user_id = author_info.user_id - res.user_link = author_info.user_link + res.creator_hash = author_info.creator_hash res.user_nickname = author_info.user_nickname - res.user_avatar = author_info.user_avatar - res.user_url_token = author_info.url_token return res def _extract_article_content(self, article: Dict) -> ZhihuContent: @@ -150,11 +148,8 @@ class ZhihuExtractor: # extract author info author_info = self._extract_content_or_comment_author(article.get("author")) - res.user_id = author_info.user_id - res.user_link = author_info.user_link + res.creator_hash = author_info.creator_hash res.user_nickname = author_info.user_nickname - res.user_avatar = author_info.user_avatar - res.user_url_token = author_info.url_token return res def _extract_zvideo_content(self, zvideo: Dict) -> ZhihuContent: @@ -184,11 +179,8 @@ class ZhihuExtractor: # extract author info author_info = self._extract_content_or_comment_author(zvideo.get("author")) - res.user_id = author_info.user_id - res.user_link = author_info.user_link + res.creator_hash = author_info.creator_hash res.user_nickname = author_info.user_nickname - res.user_avatar = author_info.user_avatar - res.user_url_token = author_info.url_token return res @staticmethod @@ -207,11 +199,8 @@ class ZhihuExtractor: return res if not author.get("id"): author = author.get("member") - res.user_id = author.get("id") - res.user_link = f"{zhihu_constant.ZHIHU_URL}/people/{author.get('url_token')}" - res.user_nickname = author.get("name") - res.user_avatar = author.get("avatar_url") - res.url_token = author.get("url_token") + res.creator_hash = anonymize_user_id(author.get("id")) + res.user_nickname = mask_nickname(author.get("name")) except Exception as e : utils.logger.warning( @@ -253,7 +242,6 @@ class ZhihuExtractor: res.parent_comment_id = comment.get("reply_comment_id") res.content = extract_text_from_html(comment.get("content")) res.publish_time = comment.get("created_time") - res.ip_location = self._extract_comment_ip_location(comment.get("comment_tag", [])) res.sub_comment_count = comment.get("child_comment_count") res.like_count = comment.get("like_count") if comment.get("like_count") else 0 res.dislike_count = comment.get("dislike_count") if comment.get("dislike_count") else 0 @@ -262,10 +250,8 @@ class ZhihuExtractor: # extract author info author_info = self._extract_content_or_comment_author(comment.get("author")) - res.user_id = author_info.user_id - res.user_link = author_info.user_link + res.creator_hash = author_info.creator_hash res.user_nickname = author_info.user_nickname - res.user_avatar = author_info.user_avatar return res @staticmethod @@ -352,13 +338,8 @@ class ZhihuExtractor: return None res = ZhihuCreator() - res.user_id = creator_info.get("id") - res.user_link = f"{zhihu_constant.ZHIHU_URL}/people/{user_url_token}" - res.user_nickname = creator_info.get("name") - res.user_avatar = creator_info.get("avatarUrl") - res.url_token = creator_info.get("urlToken") or user_url_token - res.gender = self._foramt_gender_text(creator_info.get("gender")) - res.ip_location = creator_info.get("ipInfo") + res.creator_hash = anonymize_user_id(creator_info.get("id")) + res.user_nickname = mask_nickname(creator_info.get("name")) res.follows = creator_info.get("followingCount") res.fans = creator_info.get("followerCount") res.anwser_count = creator_info.get("answerCount") diff --git a/model/m_baidu_tieba.py b/model/m_baidu_tieba.py index f537b9c..c068c38 100644 --- a/model/m_baidu_tieba.py +++ b/model/m_baidu_tieba.py @@ -33,14 +33,12 @@ class TiebaNote(BaseModel): desc: str = Field(default="", description="Post description") note_url: str = Field(..., description="Post link") publish_time: str = Field(default="", description="Publish time") - user_link: str = Field(default="", description="User homepage link") - user_nickname: str = Field(default="", description="User nickname") - user_avatar: str = Field(default="", description="User avatar URL") + creator_hash: str = Field(default="", description="创作者匿名哈希(不存原始用户链接)") + user_nickname: str = Field(default="", description="User nickname (已脱敏)") tieba_name: str = Field(..., description="Tieba name") tieba_link: str = Field(..., description="Tieba link") total_replay_num: int = Field(default=0, description="Total reply count") total_replay_page: int = Field(default=0, description="Total reply pages") - ip_location: Optional[str] = Field(default="", description="IP location") source_keyword: str = Field(default="", description="Source keyword") @@ -52,11 +50,9 @@ class TiebaComment(BaseModel): comment_id: str = Field(..., description="Comment ID") parent_comment_id: str = Field(default="", description="Parent comment ID") content: str = Field(..., description="Comment content") - user_link: str = Field(default="", description="User homepage link") - user_nickname: str = Field(default="", description="User nickname") - user_avatar: str = Field(default="", description="User avatar URL") + creator_hash: str = Field(default="", description="创作者匿名哈希(不存原始用户链接)") + user_nickname: str = Field(default="", description="User nickname (已脱敏)") publish_time: str = Field(default="", description="Publish time") - ip_location: Optional[str] = Field(default="", description="IP location") sub_comment_count: int = Field(default=0, description="Sub-comment count") note_id: str = Field(..., description="Post ID") note_url: str = Field(..., description="Post link") @@ -67,14 +63,10 @@ class TiebaComment(BaseModel): class TiebaCreator(BaseModel): """ - Baidu Tieba creator + Baidu Tieba creator（教学版：个人资料不再落库，仅作内存对象） """ - user_id: str = Field(..., description="User ID") - user_name: str = Field(..., description="Username") - nickname: str = Field(..., description="User nickname") - gender: str = Field(default="", description="User gender") - avatar: str = Field(..., description="User avatar URL") - ip_location: Optional[str] = Field(default="", description="IP location") + creator_hash: str = Field(default="", description="创作者匿名哈希(不存原始用户链接)") + user_nickname: str = Field(default="", description="User nickname (已脱敏)") follows: int = Field(default=0, description="Follows count") fans: int = Field(default=0, description="Fans count") registration_duration: str = Field(default="", description="Registration duration") diff --git a/model/m_zhihu.py b/model/m_zhihu.py index a17e6d4..6b8dcf6 100644 --- a/model/m_zhihu.py +++ b/model/m_zhihu.py @@ -19,8 +19,6 @@ # -*- coding: utf-8 -*- -from typing import Optional - from pydantic import BaseModel, Field @@ -40,12 +38,8 @@ class ZhihuContent(BaseModel): voteup_count: int = Field(default=0, description="Upvote count") comment_count: int = Field(default=0, description="Comment count") source_keyword: str = Field(default="", description="Source keyword") - - user_id: str = Field(default="", description="User ID") - user_link: str = Field(default="", description="User homepage link") - user_nickname: str = Field(default="", description="User nickname") - user_avatar: str = Field(default="", description="User avatar URL") - user_url_token: str = Field(default="", description="User url_token") + creator_hash: str = Field(default="", description="Creator anonymized hash") + user_nickname: str = Field(default="", description="User nickname (masked)") class ZhihuComment(BaseModel): @@ -57,30 +51,21 @@ class ZhihuComment(BaseModel): parent_comment_id: str = Field(default="", description="Parent comment ID") content: str = Field(default="", description="Comment content") publish_time: int = Field(default=0, description="Publish time") - ip_location: Optional[str] = Field(default="", description="IP location") sub_comment_count: int = Field(default=0, description="Sub-comment count") like_count: int = Field(default=0, description="Like count") dislike_count: int = Field(default=0, description="Dislike count") content_id: str = Field(default="", description="Content ID") content_type: str = Field(default="", description="Content type (article | answer | zvideo)") - - user_id: str = Field(default="", description="User ID") - user_link: str = Field(default="", description="User homepage link") - user_nickname: str = Field(default="", description="User nickname") - user_avatar: str = Field(default="", description="User avatar URL") + creator_hash: str = Field(default="", description="Creator anonymized hash") + user_nickname: str = Field(default="", description="User nickname (masked)") class ZhihuCreator(BaseModel): """ - Zhihu creator + Zhihu creator (in-memory only; personal profile is no longer persisted) """ - user_id: str = Field(default="", description="User ID") - user_link: str = Field(default="", description="User homepage link") - user_nickname: str = Field(default="", description="User nickname") - user_avatar: str = Field(default="", description="User avatar URL") - url_token: str = Field(default="", description="User url_token") - gender: str = Field(default="", description="User gender") - ip_location: Optional[str] = Field(default="", description="IP location") + creator_hash: str = Field(default="", description="Creator anonymized hash") + user_nickname: str = Field(default="", description="User nickname (masked)") follows: int = Field(default=0, description="Follows count") fans: int = Field(default=0, description="Fans count") anwser_count: int = Field(default=0, description="Answer count") diff --git a/store/bilibili/__init__.py b/store/bilibili/__init__.py index 5419177..b5eb921 100644 --- a/store/bilibili/__init__.py +++ b/store/bilibili/__init__.py @@ -26,6 +26,7 @@ from typing import List import config from var import source_keyword_var +from tools.user_hash import anonymize_user_id, mask_nickname from ._store_impl import * from .bilibilli_store_media import * @@ -62,9 +63,8 @@ async def update_bilibili_video(video_item: Dict): "title": video_item_view.get("title", "")[:500], "desc": video_item_view.get("desc", "")[:500], "create_time": video_item_view.get("pubdate"), - "user_id": str(video_user_info.get("mid")), - "nickname": video_user_info.get("name"), - "avatar": video_user_info.get("face", ""), + "creator_hash": anonymize_user_id(video_user_info.get("mid")), # 创作者匿名哈希(不存原始 mid) + "nickname": mask_nickname(video_user_info.get("name")), # 用户昵称(已脱敏) "liked_count": str(video_item_stat.get("like", "")), "disliked_count": str(video_item_stat.get("dislike", "")), "video_play_count": str(video_item_stat.get("view", "")), @@ -83,22 +83,8 @@ async def update_bilibili_video(video_item: Dict): async def update_up_info(video_item: Dict): - video_item_card_list: Dict = video_item.get("Card") - video_item_card: Dict = video_item_card_list.get("card") - saver_up_info = { - "user_id": str(video_item_card.get("mid")), - "nickname": video_item_card.get("name"), - "sex": video_item_card.get("sex"), - "sign": video_item_card.get("sign"), - "avatar": video_item_card.get("face"), - "last_modify_ts": utils.get_current_timestamp(), - "total_fans": video_item_card.get("fans"), - "total_liked": video_item_card_list.get("like_num"), - "user_rank": video_item_card.get("level_info").get("current_level"), - "is_official": video_item_card.get("official_verify").get("type"), - } - utils.logger.info(f"[store.bilibili.update_up_info] bilibili user_id:{video_item_card.get('mid')}") - await BiliStoreFactory.create_store().store_creator(creator=saver_up_info) + # 教学版：UP 主个人资料(昵称/性别/签名/头像/粉丝数等)不再落库，防骚扰。 + return async def batch_update_bilibili_video_comments(video_id: str, comments: List[Dict]): @@ -120,11 +106,8 @@ async def update_bilibili_video_comment(video_id: str, comment_item: Dict): "create_time": comment_item.get("ctime"), "video_id": str(video_id), "content": content.get("message"), - "user_id": user_info.get("mid"), - "nickname": user_info.get("uname"), - "sex": user_info.get("sex"), - "sign": user_info.get("sign"), - "avatar": user_info.get("avatar"), + "creator_hash": anonymize_user_id(user_info.get("mid")), # 创作者匿名哈希(不存原始 mid) + "nickname": mask_nickname(user_info.get("uname")), # 用户昵称(已脱敏) "sub_comment_count": str(comment_item.get("rcount", 0)), "like_count": like_count, "last_modify_ts": utils.get_current_timestamp(), @@ -149,29 +132,13 @@ async def store_video(aid, video_content, extension_file_name): async def batch_update_bilibili_creator_fans(creator_info: Dict, fans_list: List[Dict]): - if not fans_list: - return - for fan_item in fans_list: - fan_info: Dict = { - "id": fan_item.get("mid"), - "name": fan_item.get("uname"), - "sign": fan_item.get("sign"), - "avatar": fan_item.get("face"), - } - await update_bilibili_creator_contact(creator_info=creator_info, fan_info=fan_info) + # 教学版：不再采集/存储粉丝列表(其他用户的个人信息)，防骚扰。 + return async def batch_update_bilibili_creator_followings(creator_info: Dict, followings_list: List[Dict]): - if not followings_list: - return - for following_item in followings_list: - following_info: Dict = { - "id": following_item.get("mid"), - "name": following_item.get("uname"), - "sign": following_item.get("sign"), - "avatar": following_item.get("face"), - } - await update_bilibili_creator_contact(creator_info=following_info, fan_info=creator_info) + # 教学版：不再采集/存储关注列表(其他用户的个人信息)，防骚扰。 + return async def batch_update_bilibili_creator_dynamics(creator_info: Dict, dynamics_list: List[Dict]): @@ -201,26 +168,15 @@ async def batch_update_bilibili_creator_dynamics(creator_info: Dict, dynamics_li async def update_bilibili_creator_contact(creator_info: Dict, fan_info: Dict): - save_contact_item = { - "up_id": creator_info["id"], - "fan_id": fan_info["id"], - "up_name": creator_info["name"], - "fan_name": fan_info["name"], - "up_sign": creator_info["sign"], - "fan_sign": fan_info["sign"], - "up_avatar": creator_info["avatar"], - "fan_avatar": fan_info["avatar"], - "last_modify_ts": utils.get_current_timestamp(), - } - - await BiliStoreFactory.create_store().store_contact(contact_item=save_contact_item) + # 教学版：UP-粉丝关系表已移除，不再存储联系人信息。 + return async def update_bilibili_creator_dynamic(creator_info: Dict, dynamic_info: Dict): save_dynamic_item = { "dynamic_id": dynamic_info["dynamic_id"], - "user_id": creator_info["id"], - "user_name": creator_info["name"], + "creator_hash": anonymize_user_id(creator_info.get("id")), # 创作者匿名哈希(不存原始 ID) + "user_name": mask_nickname(creator_info.get("name")), # 用户名称(已脱敏) "text": dynamic_info["text"], "type": dynamic_info["type"], "pub_ts": dynamic_info["pub_ts"], diff --git a/store/bilibili/_store_impl.py b/store/bilibili/_store_impl.py index ed220b0..feaf280 100644 --- a/store/bilibili/_store_impl.py +++ b/store/bilibili/_store_impl.py @@ -36,7 +36,7 @@ from sqlalchemy.orm import sessionmaker import config from base.base_crawler import AbstractStore from database.db_session import get_session -from database.models import BilibiliVideoComment, BilibiliVideo, BilibiliUpInfo, BilibiliUpDynamic, BilibiliContactInfo +from database.models import BilibiliVideoComment, BilibiliVideo, BilibiliUpDynamic from tools.async_file_writer import AsyncFileWriter from tools import utils, words from var import crawler_type_var @@ -130,7 +130,6 @@ class BiliDbStoreImplement(AbstractStore): """ video_id = int(content_item.get("video_id")) content_item["video_id"] = video_id - content_item["user_id"] = int(content_item.get("user_id", 0) or 0) content_item["liked_count"] = int(content_item.get("liked_count", 0) or 0) content_item["create_time"] = int(content_item.get("create_time", 0) or 0) @@ -179,60 +178,12 @@ class BiliDbStoreImplement(AbstractStore): await session.commit() async def store_creator(self, creator: Dict): - """ - Bilibili creator DB storage implementation - Args: - creator: creator item dict - """ - creator_id = int(creator.get("user_id")) - creator["user_id"] = creator_id - creator["total_fans"] = int(creator.get("total_fans", 0) or 0) - creator["total_liked"] = int(creator.get("total_liked", 0) or 0) - creator["user_rank"] = int(creator.get("user_rank", 0) or 0) - creator["is_official"] = int(creator.get("is_official", 0) or 0) - - async with get_session() as session: - result = await session.execute(select(BilibiliUpInfo).where(BilibiliUpInfo.user_id == creator_id)) - creator_detail = result.scalar_one_or_none() - - if not creator_detail: - creator["add_ts"] = utils.get_current_timestamp() - creator["last_modify_ts"] = utils.get_current_timestamp() - new_creator = BilibiliUpInfo(**creator) - session.add(new_creator) - else: - creator["last_modify_ts"] = utils.get_current_timestamp() - for key, value in creator.items(): - setattr(creator_detail, key, value) - await session.commit() + # 教学版：UP 主个人资料不再落库 + pass async def store_contact(self, contact_item: Dict): - """ - Bilibili contact DB storage implementation - Args: - contact_item: contact item dict - """ - up_id = int(contact_item.get("up_id")) - fan_id = int(contact_item.get("fan_id")) - contact_item["up_id"] = up_id - contact_item["fan_id"] = fan_id - - async with get_session() as session: - result = await session.execute( - select(BilibiliContactInfo).where(BilibiliContactInfo.up_id == up_id, BilibiliContactInfo.fan_id == fan_id) - ) - contact_detail = result.scalar_one_or_none() - - if not contact_detail: - contact_item["add_ts"] = utils.get_current_timestamp() - contact_item["last_modify_ts"] = utils.get_current_timestamp() - new_contact = BilibiliContactInfo(**contact_item) - session.add(new_contact) - else: - contact_item["last_modify_ts"] = utils.get_current_timestamp() - for key, value in contact_item.items(): - setattr(contact_detail, key, value) - await session.commit() + # 教学版：UP-粉丝关系表已移除，不再存储联系人信息 + pass async def store_dynamic(self, dynamic_item): """ @@ -421,21 +372,8 @@ class BiliMongoStoreImplement(AbstractStore): utils.logger.info(f"[BiliMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB") async def store_creator(self, creator_item: Dict): - """ - Store UP master information to MongoDB - Args: - creator_item: UP master data - """ - user_id = creator_item.get("user_id") - if not user_id: - return - - await self.mongo_store.save_or_update( - collection_suffix="creators", - query={"user_id": user_id}, - data=creator_item - ) - utils.logger.info(f"[BiliMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB") + # 教学版：UP 主个人资料不再落库 + pass class BiliExcelStoreImplement: diff --git a/store/douyin/__init__.py b/store/douyin/__init__.py index 9e4c21c..a1e9207 100644 --- a/store/douyin/__init__.py +++ b/store/douyin/__init__.py @@ -25,6 +25,7 @@ from typing import List import config from var import source_keyword_var +from tools.user_hash import anonymize_user_id, mask_nickname from ._store_impl import * from .douyin_store_media import * @@ -164,18 +165,12 @@ async def update_douyin_aweme(aweme_item: Dict): "title": aweme_item.get("desc", ""), "desc": aweme_item.get("desc", ""), "create_time": aweme_item.get("create_time"), - "user_id": user_info.get("uid"), - "sec_uid": user_info.get("sec_uid"), - "short_user_id": user_info.get("short_id"), - "user_unique_id": user_info.get("unique_id"), - "user_signature": user_info.get("signature"), - "nickname": user_info.get("nickname"), - "avatar": user_info.get("avatar_thumb", {}).get("url_list", [""])[0], + "creator_hash": anonymize_user_id(user_info.get("uid")), # 创作者匿名哈希(不存原始 uid) + "nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏) "liked_count": str(interact_info.get("digg_count")), "collected_count": str(interact_info.get("collect_count")), "comment_count": str(interact_info.get("comment_count")), "share_count": str(interact_info.get("share_count")), - "ip_location": aweme_item.get("ip_label", ""), "last_modify_ts": utils.get_current_timestamp(), "aweme_url": f"https://www.douyin.com/video/{aweme_id}", "cover_url": _extract_content_cover_url(aweme_item), @@ -203,20 +198,13 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict): user_info = comment_item.get("user", {}) comment_id = comment_item.get("cid") parent_comment_id = comment_item.get("reply_id", "0") - avatar_info = (user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get("avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {}) save_comment_item = { "comment_id": comment_id, "create_time": comment_item.get("create_time"), - "ip_location": comment_item.get("ip_label", ""), "aweme_id": aweme_id, "content": comment_item.get("text"), - "user_id": user_info.get("uid"), - "sec_uid": user_info.get("sec_uid"), - "short_user_id": user_info.get("short_id"), - "user_unique_id": user_info.get("unique_id"), - "user_signature": user_info.get("signature"), - "nickname": user_info.get("nickname"), - "avatar": avatar_info.get("url_list", [""])[0], + "creator_hash": anonymize_user_id(user_info.get("uid")), # 创作者匿名哈希(不存原始 uid) + "nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏) "sub_comment_count": str(comment_item.get("reply_comment_total", 0)), "like_count": (comment_item.get("digg_count") if comment_item.get("digg_count") else 0), "last_modify_ts": utils.get_current_timestamp(), @@ -229,24 +217,8 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict): async def save_creator(user_id: str, creator: Dict): - user_info = creator.get("user", {}) - gender_map = {0: "Unknown", 1: "Male", 2: "Female"} - avatar_uri = user_info.get("avatar_300x300", {}).get("uri") - local_db_item = { - "user_id": user_id, - "nickname": user_info.get("nickname"), - "gender": gender_map.get(user_info.get("gender"), "Unknown"), - "avatar": f"https://p3-pc.douyinpic.com/img/{avatar_uri}" + r"~c5_300x300.jpeg?from=2956013662", - "desc": user_info.get("signature"), - "ip_location": user_info.get("ip_location"), - "follows": user_info.get("following_count", 0), - "fans": user_info.get("max_follower_count", 0), - "interaction": user_info.get("total_favorited", 0), - "videos_count": user_info.get("aweme_count", 0), - "last_modify_ts": utils.get_current_timestamp(), - } - utils.logger.info(f"[store.douyin.save_creator] creator:{local_db_item}") - await DouyinStoreFactory.create_store().store_creator(local_db_item) + # 教学版：创作者个人资料(昵称/性别/头像/签名/IP/粉丝数等)不再落库，防骚扰。 + return async def update_dy_aweme_image(aweme_id, pic_content, extension_file_name): diff --git a/store/douyin/_store_impl.py b/store/douyin/_store_impl.py index 93f7fa8..bc32f34 100644 --- a/store/douyin/_store_impl.py +++ b/store/douyin/_store_impl.py @@ -33,7 +33,7 @@ from sqlalchemy import select import config from base.base_crawler import AbstractStore from database.db_session import get_session -from database.models import DouyinAweme, DouyinAwemeComment, DyCreator +from database.models import DouyinAweme, DouyinAwemeComment from tools import utils, words from tools.async_file_writer import AsyncFileWriter from var import crawler_type_var @@ -133,24 +133,8 @@ class DouyinDbStoreImplement(AbstractStore): await session.commit() async def store_creator(self, creator: Dict): - """ - Douyin creator DB storage implementation - Args: - creator: creator dict - """ - user_id = creator.get("user_id") - async with get_session() as session: - result = await session.execute(select(DyCreator).where(DyCreator.user_id == user_id)) - user_detail = result.scalar_one_or_none() - - if not user_detail: - creator["add_ts"] = utils.get_current_timestamp() - new_creator = DyCreator(**creator) - session.add(new_creator) - else: - for key, value in creator.items(): - setattr(user_detail, key, value) - await session.commit() + # 教学版：创作者个人资料不再落库 + pass class DouyinJsonStoreImplement(AbstractStore): @@ -275,21 +259,8 @@ class DouyinMongoStoreImplement(AbstractStore): utils.logger.info(f"[DouyinMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB") async def store_creator(self, creator_item: Dict): - """ - Store creator information to MongoDB - Args: - creator_item: Creator data - """ - user_id = creator_item.get("user_id") - if not user_id: - return - - await self.mongo_store.save_or_update( - collection_suffix="creators", - query={"user_id": user_id}, - data=creator_item - ) - utils.logger.info(f"[DouyinMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB") + # 教学版：创作者个人资料不再落库 + pass class DouyinExcelStoreImplement: diff --git a/store/kuaishou/__init__.py b/store/kuaishou/__init__.py index bbd0981..c8034de 100644 --- a/store/kuaishou/__init__.py +++ b/store/kuaishou/__init__.py @@ -26,6 +26,7 @@ from typing import List import config from var import source_keyword_var +from tools.user_hash import anonymize_user_id, mask_nickname from ._store_impl import * @@ -63,9 +64,8 @@ async def update_kuaishou_video(video_item: Dict): "title": photo_info.get("caption", "")[:500], "desc": photo_info.get("caption", "")[:500], "create_time": photo_info.get("timestamp"), - "user_id": user_info.get("id"), - "nickname": user_info.get("name"), - "avatar": user_info.get("headerUrl", ""), + "creator_hash": anonymize_user_id(user_info.get("id")), # 创作者匿名哈希(不存原始 user_id) + "nickname": mask_nickname(user_info.get("name")), # 用户昵称(已脱敏) "liked_count": str(photo_info.get("realLikeCount")), "viewd_count": str(photo_info.get("viewCount")), "last_modify_ts": utils.get_current_timestamp(), @@ -97,11 +97,10 @@ async def update_ks_video_comment(video_id: str, comment_item: Dict): "create_time": comment_item.get("timestamp"), "video_id": video_id, "content": comment_item.get("content"), - # V2: author_id, Old: authorId - "user_id": comment_item.get("author_id") or comment_item.get("authorId"), - # V2: author_name, Old: authorName - "nickname": comment_item.get("author_name") or comment_item.get("authorName"), - "avatar": comment_item.get("headurl"), + # 创作者匿名哈希(不存原始 user_id)：V2: author_id, Old: authorId + "creator_hash": anonymize_user_id(comment_item.get("author_id") or comment_item.get("authorId")), + # 用户昵称(已脱敏)：V2: author_name, Old: authorName + "nickname": mask_nickname(comment_item.get("author_name") or comment_item.get("authorName")), # V2: commentCount, Old: subCommentCount "sub_comment_count": str(comment_item.get("commentCount") or comment_item.get("subCommentCount", 0)), "last_modify_ts": utils.get_current_timestamp(), @@ -111,20 +110,5 @@ async def update_ks_video_comment(video_id: str, comment_item: Dict): await KuaishouStoreFactory.create_store().store_comment(comment_item=save_comment_item) async def save_creator(user_id: str, creator: Dict): - ownerCount = creator.get('ownerCount', {}) - profile = creator.get('profile', {}) - - local_db_item = { - 'user_id': user_id, - 'nickname': profile.get('user_name'), - 'gender': 'Female' if profile.get('gender') == "F" else 'Male', - 'avatar': profile.get('headurl'), - 'desc': profile.get('user_text'), - 'ip_location': "", - 'follows': ownerCount.get("follow"), - 'fans': ownerCount.get("fan"), - 'interaction': ownerCount.get("photo_public"), - "last_modify_ts": utils.get_current_timestamp(), - } - utils.logger.info(f"[store.kuaishou.save_creator] creator:{local_db_item}") - await KuaishouStoreFactory.create_store().store_creator(local_db_item) + # 教学版：创作者个人资料(昵称/性别/头像/签名/IP/粉丝数等)不再落库，防骚扰。 + return diff --git a/store/kuaishou/_store_impl.py b/store/kuaishou/_store_impl.py index 8e44a01..ce339b9 100644 --- a/store/kuaishou/_store_impl.py +++ b/store/kuaishou/_store_impl.py @@ -228,21 +228,8 @@ class KuaishouMongoStoreImplement(AbstractStore): utils.logger.info(f"[KuaishouMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB") async def store_creator(self, creator_item: Dict): - """ - Store creator information to MongoDB - Args: - creator_item: Creator data - """ - user_id = creator_item.get("user_id") - if not user_id: - return - - await self.mongo_store.save_or_update( - collection_suffix="creators", - query={"user_id": user_id}, - data=creator_item - ) - utils.logger.info(f"[KuaishouMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB") + # 教学版：创作者个人资料不再落库 + pass class KuaishouExcelStoreImplement: diff --git a/store/tieba/__init__.py b/store/tieba/__init__.py index ca10320..06dea76 100644 --- a/store/tieba/__init__.py +++ b/store/tieba/__init__.py @@ -121,7 +121,5 @@ async def save_creator(user_info: TiebaCreator): Returns: """ - local_db_item = user_info.model_dump() - local_db_item["last_modify_ts"] = utils.get_current_timestamp() - utils.logger.info(f"[store.tieba.save_creator] creator:{local_db_item}") - await TieBaStoreFactory.create_store().store_creator(local_db_item) + # 教学版：创作者个人资料不再落库，防骚扰。 + return diff --git a/store/tieba/_store_impl.py b/store/tieba/_store_impl.py index a259334..d1b3d79 100644 --- a/store/tieba/_store_impl.py +++ b/store/tieba/_store_impl.py @@ -35,7 +35,7 @@ from sqlalchemy.ext.asyncio import AsyncSession import config from base.base_crawler import AbstractStore -from database.models import TiebaNote, TiebaComment, TiebaCreator +from database.models import TiebaNote, TiebaComment from tools import utils, words from database.db_session import get_session from var import crawler_type_var @@ -137,23 +137,8 @@ class TieBaDbStoreImplement(AbstractStore): await session.commit() async def store_creator(self, creator: Dict): - """ - tieba content DB storage implementation - Args: - creator: creator dict - """ - user_id = creator.get("user_id") - async with get_session() as session: - stmt = select(TiebaCreator).where(TiebaCreator.user_id == user_id) - res = await session.execute(stmt) - db_creator = res.scalar_one_or_none() - if db_creator: - for key, value in creator.items(): - setattr(db_creator, key, value) - else: - db_creator = TiebaCreator(**creator) - session.add(db_creator) - await session.commit() + # 教学版：创作者个人资料不再落库 + pass class TieBaJsonStoreImplement(AbstractStore): @@ -258,20 +243,8 @@ class TieBaMongoStoreImplement(AbstractStore): utils.logger.info(f"[TieBaMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB") async def store_creator(self, creator_item: Dict): - """ - Store creator information to MongoDB - Args: - creator_item: Creator data - """ - user_id = creator_item.get("user_id") - if not user_id: - return - - await self.mongo_store.save_or_update( - collection_suffix="creators", - query={"user_id": user_id}, - data=creator_item - ) + # 教学版：创作者个人资料不再落库 + pass utils.logger.info(f"[TieBaMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB") diff --git a/store/weibo/__init__.py b/store/weibo/__init__.py index bef0fe5..bdf632b 100644 --- a/store/weibo/__init__.py +++ b/store/weibo/__init__.py @@ -25,6 +25,7 @@ import re from typing import List +from tools.user_hash import anonymize_user_id, mask_nickname from var import source_keyword_var from .weibo_store_media import * @@ -78,11 +79,13 @@ async def update_weibo_note(note_item: Dict): if not note_item: return - mblog: Dict = note_item.get("mblog") - user_info: Dict = mblog.get("user") + mblog: Dict = note_item.get("mblog") or {} + user_info: Dict = mblog.get("user") or {} note_id = mblog.get("id") content_text = mblog.get("text") clean_text = re.sub(r"<.*?>", "", content_text) + # 教学版：原始 user_id 匿名化为 creator_hash，昵称脱敏； + # 不采集头像/主页链接/性别/IP 归属地等可定位真人的信息。 save_content_item = { # Weibo information "note_id": note_id, @@ -94,14 +97,10 @@ async def update_weibo_note(note_item: Dict): "shared_count": str(mblog.get("reposts_count", 0)), "last_modify_ts": utils.get_current_timestamp(), "note_url": f"https://m.weibo.cn/detail/{note_id}", - "ip_location": mblog.get("region_name", "").replace("发布于 ", ""), - # User information - "user_id": str(user_info.get("id")), - "nickname": user_info.get("screen_name", ""), - "gender": user_info.get("gender", ""), - "profile_url": user_info.get("profile_url", ""), - "avatar": user_info.get("profile_image_url", ""), + # 创作者信息（匿名化/脱敏，不含原始 user_id/avatar/gender/profile_url/ip_location） + "creator_hash": anonymize_user_id(user_info.get("id")), + "nickname": mask_nickname(user_info.get("screen_name", "")), "source_keyword": source_keyword_var.get(), } utils.logger.info(f"[store.weibo.update_weibo_note] weibo note id:{note_id}, title:{save_content_item.get('content')[:24]} ...") @@ -137,9 +136,11 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict): if not comment_item or not note_id: return comment_id = str(comment_item.get("id")) - user_info: Dict = comment_item.get("user") + user_info: Dict = comment_item.get("user") or {} content_text = comment_item.get("text") clean_text = re.sub(r"<.*?>", "", content_text) + # 教学版：原始 user_id 匿名化为 creator_hash，昵称脱敏； + # 不采集头像/主页链接/性别/IP 归属地等可定位真人的信息。 save_comment_item = { "comment_id": comment_id, "create_time": utils.rfc2822_to_timestamp(comment_item.get("created_at")), @@ -149,15 +150,11 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict): "sub_comment_count": str(comment_item.get("total_number", 0)), "comment_like_count": str(comment_item.get("like_count", 0)), "last_modify_ts": utils.get_current_timestamp(), - "ip_location": comment_item.get("source", "").replace("来自", ""), "parent_comment_id": comment_item.get("rootid", ""), - # User information - "user_id": str(user_info.get("id")), - "nickname": user_info.get("screen_name", ""), - "gender": user_info.get("gender", ""), - "profile_url": user_info.get("profile_url", ""), - "avatar": user_info.get("profile_image_url", ""), + # 创作者信息（匿名化/脱敏，不含原始 user_id/avatar/gender/profile_url/ip_location） + "creator_hash": anonymize_user_id(user_info.get("id")), + "nickname": mask_nickname(user_info.get("screen_name", "")), } utils.logger.info(f"[store.weibo.update_weibo_note_comment] Weibo note comment: {comment_id}, content: {save_comment_item.get('content', '')[:24]} ...") await WeibostoreFactory.create_store().store_comment(comment_item=save_comment_item) @@ -180,6 +177,8 @@ async def update_weibo_note_image(picid: str, pic_content, extension_file_name): async def save_creator(user_id: str, user_info: Dict): """ Save creator information to local + 教学版：为防骚扰不再采集/持久化创作者个人信息（昵称/性别/头像/简介/IP/粉丝数等）， + 此入口保留为空操作以兼容调用方。user_id 仅在调用方局部用于抓取该创作者的微博。 Args: user_id: user_info: @@ -187,17 +186,5 @@ async def save_creator(user_id: str, user_info: Dict): Returns: """ - local_db_item = { - 'user_id': user_id, - 'nickname': user_info.get('screen_name'), - 'gender': 'Female' if user_info.get('gender') == "f" else 'Male', - 'avatar': user_info.get('avatar_hd'), - 'desc': user_info.get('description'), - 'ip_location': user_info.get("source", "").replace("来自", ""), - 'follows': user_info.get('follow_count', ''), - 'fans': user_info.get('followers_count', ''), - 'tag_list': '', - "last_modify_ts": utils.get_current_timestamp(), - } - utils.logger.info(f"[store.weibo.save_creator] creator:{local_db_item}") - await WeibostoreFactory.create_store().store_creator(local_db_item) + # 教学版：创作者个人信息均不采集不持久化 + return diff --git a/store/weibo/_store_impl.py b/store/weibo/_store_impl.py index db75bf0..45b2ab9 100644 --- a/store/weibo/_store_impl.py +++ b/store/weibo/_store_impl.py @@ -35,7 +35,7 @@ from sqlalchemy.ext.asyncio import AsyncSession import config from base.base_crawler import AbstractStore -from database.models import WeiboCreator, WeiboNote, WeiboNoteComment +from database.models import WeiboNote, WeiboNoteComment from tools import utils, words from tools.async_file_writer import AsyncFileWriter from database.db_session import get_session @@ -58,6 +58,13 @@ def calculate_number_of_files(file_store_path: str) -> int: return 1 +def _filter_model_fields(model_cls, item: Dict) -> Dict: + """只保留目标 ORM 模型已有的列，避免把已删除/多余字段（如 avatar/gender/ + profile_url/ip_location/user_id）传给 ORM 构造而报错。教学版兜底保护。""" + allowed = {col.name for col in model_cls.__table__.columns} + return {k: v for k, v in item.items() if k in allowed} + + class WeiboCsvStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -88,13 +95,14 @@ class WeiboCsvStoreImplement(AbstractStore): async def store_creator(self, creator: Dict): """ Weibo creator CSV storage implementation + 教学版：不采集/持久化创作者个人信息，空操作。 Args: creator: Returns: """ - await self.writer.write_to_csv(item_type="creators", item=creator) + pass class WeiboDbStoreImplement(AbstractStore): @@ -108,6 +116,8 @@ class WeiboDbStoreImplement(AbstractStore): Returns: """ + # 教学版兜底：过滤掉已删除/多余字段，确保不会把 user_id/avatar 等传给 ORM + content_item = _filter_model_fields(WeiboNote, content_item) note_id = int(content_item.get("note_id")) content_item["note_id"] = note_id async with get_session() as session: @@ -135,6 +145,8 @@ class WeiboDbStoreImplement(AbstractStore): Returns: """ + # 教学版兜底：过滤掉已删除/多余字段，确保不会把 user_id/avatar 等传给 ORM + comment_item = _filter_model_fields(WeiboNoteComment, comment_item) comment_id = int(comment_item.get("comment_id")) comment_item["comment_id"] = comment_id comment_item["note_id"] = int(comment_item.get("note_id", 0) or 0) @@ -162,29 +174,14 @@ class WeiboDbStoreImplement(AbstractStore): async def store_creator(self, creator: Dict): """ Weibo creator DB storage implementation + 教学版：不采集/持久化创作者个人信息，空操作（WeiboCreator 表已删除）。 Args: creator: Returns: """ - user_id = int(creator.get("user_id")) - creator["user_id"] = user_id - async with get_session() as session: - stmt = select(WeiboCreator).where(WeiboCreator.user_id == user_id) - res = await session.execute(stmt) - db_creator = res.scalar_one_or_none() - if db_creator: - db_creator.last_modify_ts = utils.get_current_timestamp() - for key, value in creator.items(): - if hasattr(db_creator, key): - setattr(db_creator, key, value) - else: - creator["add_ts"] = utils.get_current_timestamp() - creator["last_modify_ts"] = utils.get_current_timestamp() - db_creator = WeiboCreator(**creator) - session.add(db_creator) - await session.commit() + pass class WeiboJsonStoreImplement(AbstractStore): @@ -217,13 +214,14 @@ class WeiboJsonStoreImplement(AbstractStore): async def store_creator(self, creator: Dict): """ creator JSON storage implementation + 教学版：不采集/持久化创作者个人信息，空操作。 Args: creator: Returns: """ - await self.writer.write_single_item_to_json(item_type="creators", item=creator) + pass class WeiboJsonlStoreImplement(AbstractStore): @@ -238,7 +236,8 @@ class WeiboJsonlStoreImplement(AbstractStore): await self.writer.write_to_jsonl(item_type="comments", item=comment_item) async def store_creator(self, creator: Dict): - await self.writer.write_to_jsonl(item_type="creators", item=creator) + # 教学版：不采集/持久化创作者个人信息，空操作。 + pass class WeiboSqliteStoreImplement(WeiboDbStoreImplement): @@ -291,19 +290,11 @@ class WeiboMongoStoreImplement(AbstractStore): async def store_creator(self, creator_item: Dict): """ Store creator information to MongoDB + 教学版：不采集/持久化创作者个人信息，空操作。 Args: creator_item: Creator data """ - user_id = creator_item.get("user_id") - if not user_id: - return - - await self.mongo_store.save_or_update( - collection_suffix="creators", - query={"user_id": user_id}, - data=creator_item - ) - utils.logger.info(f"[WeiboMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB") + pass class WeiboExcelStoreImplement: diff --git a/store/xhs/__init__.py b/store/xhs/__init__.py index 8796af4..31927e5 100644 --- a/store/xhs/__init__.py +++ b/store/xhs/__init__.py @@ -25,6 +25,7 @@ from typing import List import config from var import source_keyword_var +from tools.user_hash import anonymize_user_id, mask_nickname from .xhs_store_media import * from ._store_impl import * @@ -113,14 +114,12 @@ async def update_xhs_note(note_item: Dict): "video_url": video_url, # Note video url "time": note_item.get("time"), # Note publish time "last_update_time": note_item.get("last_update_time", 0), # Note last update time - "user_id": user_info.get("user_id"), # User ID - "nickname": user_info.get("nickname"), # User nickname - "avatar": user_info.get("avatar"), # User avatar + "creator_hash": anonymize_user_id(user_info.get("user_id")), # 创作者匿名哈希(不存原始 user_id) + "nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏) "liked_count": interact_info.get("liked_count"), # Like count "collected_count": interact_info.get("collected_count"), # Collection count "comment_count": interact_info.get("comment_count"), # Comment count "share_count": interact_info.get("share_count"), # Share count - "ip_location": note_item.get("ip_location", ""), # IP location "image_list": ','.join([img.get('url', '') for img in image_list]), # Image URLs "tag_list": ','.join([tag.get('name', '') for tag in tag_list if tag.get('type') == 'topic']), # Tags "last_modify_ts": utils.get_current_timestamp(), # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage) @@ -165,12 +164,10 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict): local_db_item = { "comment_id": comment_id, # Comment ID "create_time": comment_item.get("create_time"), # Comment time - "ip_location": comment_item.get("ip_location"), # IP location "note_id": note_id, # Note ID "content": comment_item.get("content"), # Comment content - "user_id": user_info.get("user_id"), # User ID - "nickname": user_info.get("nickname"), # User nickname - "avatar": user_info.get("image"), # User avatar + "creator_hash": anonymize_user_id(user_info.get("user_id")), # 创作者匿名哈希(不存原始 user_id) + "nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏) "sub_comment_count": comment_item.get("sub_comment_count", 0), # Sub-comment count "pictures": ",".join(comment_pictures), # Comment pictures "parent_comment_id": target_comment.get("id", 0), # Parent comment ID @@ -191,43 +188,8 @@ async def save_creator(user_id: str, creator: Dict): Returns: """ - user_info = creator.get('basicInfo', {}) - - follows = 0 - fans = 0 - interaction = 0 - for i in creator.get('interactions'): - if i.get('type') == 'follows': - follows = i.get('count') - elif i.get('type') == 'fans': - fans = i.get('count') - elif i.get('type') == 'interaction': - interaction = i.get('count') - - def get_gender(gender): - if gender == 1: - return 'Female' - elif gender == 0: - return 'Male' - else: - return None - - local_db_item = { - 'user_id': user_id, # User ID - 'nickname': user_info.get('nickname'), # Nickname - 'gender': get_gender(user_info.get('gender')), # Gender - 'avatar': user_info.get('images'), # Avatar - 'desc': user_info.get('desc'), # Personal description - 'ip_location': user_info.get('ipLocation'), # IP location - 'follows': follows, # Following count - 'fans': fans, # Fans count - 'interaction': interaction, # Interaction count - 'tag_list': json.dumps({tag.get('tagType'): tag.get('name') - for tag in creator.get('tags')}, ensure_ascii=False), # Tags - "last_modify_ts": utils.get_current_timestamp(), # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage) - } - utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}") - await XhsStoreFactory.create_store().store_creator(local_db_item) + # 教学版：创作者个人资料(昵称/性别/头像/IP/粉丝数等)不再落库，防骚扰。 + return async def update_xhs_note_image(note_id, pic_content, extension_file_name): diff --git a/store/xhs/_store_impl.py b/store/xhs/_store_impl.py index ade031c..bd1627b 100644 --- a/store/xhs/_store_impl.py +++ b/store/xhs/_store_impl.py @@ -30,7 +30,7 @@ from sqlalchemy.orm import Session from base.base_crawler import AbstractStore from database.db_session import get_session -from database.models import XhsNote, XhsNoteComment, XhsCreator +from database.models import XhsNote, XhsNoteComment from tools.async_file_writer import AsyncFileWriter from tools.time_util import get_current_timestamp @@ -137,10 +137,8 @@ class XhsDbStoreImplement(AbstractStore): add_ts = int(get_current_timestamp()) last_modify_ts = int(get_current_timestamp()) note = XhsNote( - user_id=content_item.get("user_id"), + creator_hash=content_item.get("creator_hash"), nickname=content_item.get("nickname"), - avatar=content_item.get("avatar"), - ip_location=content_item.get("ip_location"), add_ts=add_ts, last_modify_ts=last_modify_ts, note_id=content_item.get("note_id"), @@ -197,10 +195,8 @@ class XhsDbStoreImplement(AbstractStore): add_ts = int(get_current_timestamp()) last_modify_ts = int(get_current_timestamp()) comment = XhsNoteComment( - user_id=comment_item.get("user_id"), + creator_hash=comment_item.get("creator_hash"), nickname=comment_item.get("nickname"), - avatar=comment_item.get("avatar"), - ip_location=comment_item.get("ip_location"), add_ts=add_ts, last_modify_ts=last_modify_ts, comment_id=comment_item.get("comment_id"), @@ -231,54 +227,8 @@ class XhsDbStoreImplement(AbstractStore): return result.first() is not None async def store_creator(self, creator_item: Dict): - user_id = creator_item.get("user_id") - if not user_id: - return - async with get_session() as session: - if await self.creator_is_exist(session, user_id): - await self.update_creator(session, creator_item) - else: - await self.add_creator(session, creator_item) - - async def add_creator(self, session: AsyncSession, creator_item: Dict): - add_ts = int(get_current_timestamp()) - last_modify_ts = int(get_current_timestamp()) - creator = XhsCreator( - user_id=creator_item.get("user_id"), - nickname=creator_item.get("nickname"), - avatar=creator_item.get("avatar"), - ip_location=creator_item.get("ip_location"), - add_ts=add_ts, - last_modify_ts=last_modify_ts, - desc=creator_item.get("desc"), - gender=creator_item.get("gender"), - follows=str(creator_item.get("follows")), - fans=str(creator_item.get("fans")), - interaction=str(creator_item.get("interaction")), - tag_list=json.dumps(creator_item.get("tag_list")) - ) - session.add(creator) - - async def update_creator(self, session: AsyncSession, creator_item: Dict): - user_id = creator_item.get("user_id") - last_modify_ts = int(get_current_timestamp()) - update_data = { - "last_modify_ts": last_modify_ts, - "nickname": creator_item.get("nickname"), - "avatar": creator_item.get("avatar"), - "desc": creator_item.get("desc"), - "follows": str(creator_item.get("follows")), - "fans": str(creator_item.get("fans")), - "interaction": str(creator_item.get("interaction")), - "tag_list": json.dumps(creator_item.get("tag_list")) - } - stmt = update(XhsCreator).where(XhsCreator.user_id == user_id).values(**update_data) - await session.execute(stmt) - - async def creator_is_exist(self, session: AsyncSession, user_id: str) -> bool: - stmt = select(XhsCreator).where(XhsCreator.user_id == user_id) - result = await session.execute(stmt) - return result.first() is not None + # 教学版：创作者个人资料不再落库 + pass async def get_all_content(self) -> List[Dict]: async with get_session() as session: @@ -345,16 +295,8 @@ class XhsMongoStoreImplement(AbstractStore): Args: creator_item: Creator data """ - user_id = creator_item.get("user_id") - if not user_id: - return - - await self.mongo_store.save_or_update( - collection_suffix="creators", - query={"user_id": user_id}, - data=creator_item - ) - utils.logger.info(f"[XhsMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB") + # 教学版：创作者个人资料不再落库 + pass class XhsExcelStoreImplement: diff --git a/store/zhihu/_store_impl.py b/store/zhihu/_store_impl.py index a2ff628..b0bc661 100644 --- a/store/zhihu/_store_impl.py +++ b/store/zhihu/_store_impl.py @@ -36,7 +36,7 @@ from sqlalchemy.ext.asyncio import AsyncSession import config from base.base_crawler import AbstractStore from database.db_session import get_session -from database.models import ZhihuContent, ZhihuComment, ZhihuCreator +from database.models import ZhihuContent, ZhihuComment from tools import utils, words from var import crawler_type_var from tools.async_file_writer import AsyncFileWriter @@ -85,15 +85,8 @@ class ZhihuCsvStoreImplement(AbstractStore): await self.writer.write_to_csv(item_type="comments", item=comment_item) async def store_creator(self, creator: Dict): - """ - Zhihu content CSV storage implementation - Args: - creator: creator dict - - Returns: - - """ - await self.writer.write_to_csv(item_type="creators", item=creator) + """Creator profile is no longer persisted (teaching version: anti-harassment).""" + pass class ZhihuDbStoreImplement(AbstractStore): @@ -142,26 +135,8 @@ class ZhihuDbStoreImplement(AbstractStore): await session.commit() async def store_creator(self, creator: Dict): - """ - Zhihu content DB storage implementation - Args: - creator: creator dict - """ - user_id = creator.get("user_id") - async with get_session() as session: - stmt = select(ZhihuCreator).where(ZhihuCreator.user_id == user_id) - result = await session.execute(stmt) - existing_creator = result.scalars().first() - if existing_creator: - for key, value in creator.items(): - if hasattr(existing_creator, key): - setattr(existing_creator, key, value) - else: - if "add_ts" not in creator: - creator["add_ts"] = utils.get_current_timestamp() - new_creator = ZhihuCreator(**creator) - session.add(new_creator) - await session.commit() + """Creator profile is no longer persisted (teaching version: anti-harassment).""" + pass class ZhihuJsonStoreImplement(AbstractStore): @@ -192,15 +167,8 @@ class ZhihuJsonStoreImplement(AbstractStore): await self.writer.write_single_item_to_json(item_type="comments", item=comment_item) async def store_creator(self, creator: Dict): - """ - Zhihu content JSON storage implementation - Args: - creator: creator dict - - Returns: - - """ - await self.writer.write_single_item_to_json(item_type="creators", item=creator) + """Creator profile is no longer persisted (teaching version: anti-harassment).""" + pass class ZhihuJsonlStoreImplement(AbstractStore): @@ -215,7 +183,8 @@ class ZhihuJsonlStoreImplement(AbstractStore): await self.writer.write_to_jsonl(item_type="comments", item=comment_item) async def store_creator(self, creator: Dict): - await self.writer.write_to_jsonl(item_type="creators", item=creator) + """Creator profile is no longer persisted (teaching version: anti-harassment).""" + pass class ZhihuSqliteStoreImplement(ZhihuDbStoreImplement): @@ -266,21 +235,8 @@ class ZhihuMongoStoreImplement(AbstractStore): utils.logger.info(f"[ZhihuMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB") async def store_creator(self, creator_item: Dict): - """ - Store creator information to MongoDB - Args: - creator_item: Creator data - """ - user_id = creator_item.get("user_id") - if not user_id: - return - - await self.mongo_store.save_or_update( - collection_suffix="creators", - query={"user_id": user_id}, - data=creator_item - ) - utils.logger.info(f"[ZhihuMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB") + """Creator profile is no longer persisted (teaching version: anti-harassment).""" + pass class ZhihuExcelStoreImplement: diff --git a/tests/test_douyin_no_user_info.py b/tests/test_douyin_no_user_info.py new file mode 100644 index 0000000..bc08cba --- /dev/null +++ b/tests/test_douyin_no_user_info.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +""" +教学版回归测试(抖音 douyin):确保 store/douyin 存储链路不再持久化可定位真人的 +用户个人信息。 + +教学版约定:用户 ID(uid/sec_uid/short_user_id/user_unique_id)/IP/头像/签名/性别 +一律不持久化;原始 uid 经 tools.user_hash.anonymize_user_id 转成 creator_hash +(sha256 截断 16 位)写入;昵称保留但经 mask_nickname 中间脱敏。desc(作品描述)是 +内容字段,保留。 + +覆盖: +1. test_douyin_aweme_masks_user_info + - mock aweme_item(含 author.uid/sec_uid/short_id/unique_id/nickname/ + avatar_thumb.url_list/signature、aweme_item.ip_label、statistics、video/ + music/images 等内容字段) + - FakeStore 捕获 update_douyin_aweme 产出的存储 dict + - 断言:不含禁用键;含 creator_hash(≠原 uid);nickname 脱敏≠原文; + 原始敏感值不在任何存储值中;内容字段正常提取 +2. test_douyin_comment_masks_user_info + - 同上,针对 update_dy_aweme_comment(含嵌套 user.* / comment_item.ip_label / + cid / text / image_list 等) +3. test_douyin_store_end_to_end_sqlite + - FakeStore 捕获真实 dict → DouyinAweme(**captured) 构造(ORM 列校验: + dict 多了已删列会 TypeError) + - 端到端:monkeypatch db_session 为内存 SQLite engine + create_all, + 走真实 DouyinDbStoreImplement.store_content(含 int(aweme_id) / if title + 判断 / DouyinAweme(**content_item)),查询回读,验证 dict key 与删列后 ORM + 完全对得上 +""" +import asyncio + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.pool import StaticPool + +import config +import store.douyin as ds +from database import db_session +from database.models import Base, DouyinAweme, DouyinAwemeComment +from tools.user_hash import anonymize_user_id, mask_nickname + + +# 抖音教学版禁用字段(键):不得作为存储 dict 的 key 出现。 +FORBIDDEN_KEYS = { + "user_id", "sec_uid", "short_user_id", "user_unique_id", + "avatar", "user_signature", "ip_location", +} + + +# ----------------------------- mock payload ----------------------------- + +def _build_aweme_item() -> dict: + """贴近真实抖音结构的 mock aweme_item。 + 含会被拍平的用户字段(uid/sec_uid/short_id/unique_id/avatar/signature)与 + ip_label,这些必须不落库;同时含 statistics/video/music/images 等内容字段, + 让 _extract_* 辅助函数返回非空值。""" + return { + "aweme_id": "7234567890123456", + "aweme_type": 0, + "desc": "这是一个作品描述,同时作为 title", + "create_time": 1700000000, + "ip_label": "上海", # IP 归属地,禁用 + "author": { + "uid": "9876543210", + "sec_uid": "MS4wLjABAAAASecretSecUidForTest", + "short_id": "88877766", + "unique_id": "creator_unique_abc", + "nickname": "抖音创作者", + "avatar_thumb": {"url_list": ["http://x/avatar_thumb.jpg"]}, + "avatar_medium": {"url_list": ["http://x/avatar_medium.jpg"]}, + "signature": "这是创作者个人签名内容", + }, + "statistics": { + "digg_count": 100, + "collect_count": 5, + "comment_count": 20, + "share_count": 3, + }, + "video": { + "raw_cover": {"url_list": ["", "http://x/cover.jpg"]}, + "origin_cover": {"url_list": ["", "http://x/cover2.jpg"]}, + "play_addr_h264": {"url_list": ["", "http://x/video_h264.mp4"]}, + "play_addr_256": {"url_list": ["", "http://x/video_256.mp4"]}, + "play_addr": {"url_list": ["", "http://x/video.mp4"]}, + }, + "music": { + "play_url": {"uri": "http://x/music.mp3"}, + }, + "images": [ + {"url_list": ["", "http://x/note_img1.jpg"]}, + {"url_list": ["", "http://x/note_img2.jpg"]}, + ], + } + + +def _build_comment_item() -> dict: + """贴近真实抖音结构的 mock comment_item。aweme_id 须与传入 + update_dy_aweme_comment 的 aweme_id 一致。""" + return { + "aweme_id": "7234567890123456", + "cid": "1111111111", + "text": "这是一条评论内容", + "create_time": 1700000001, + "reply_id": "0", + "reply_comment_total": 2, + "digg_count": 5, + "ip_label": "北京", # IP 归属地,禁用 + "user": { + "uid": "555666777", + "sec_uid": "MS4wLjABBBBCommentSecUid", + "short_id": "22233344", + "unique_id": "commenter_unique_xyz", + "nickname": "评论员小王", + "avatar_medium": {"url_list": ["http://x/cavatar.jpg"]}, + "signature": "评论员个人签名内容", + }, + "image_list": [ + {"origin_url": {"url_list": ["", "http://x/cimg.jpg"]}}, + ], + } + + +# ----------------------------- 辅助 ----------------------------- + +class _FakeStore: + """捕获存储 dict,不真正落库。""" + + def __init__(self): + self.contents = [] + self.comments = [] + + async def store_content(self, content_item): + self.contents.append(dict(content_item)) + + async def store_comment(self, comment_item): + self.comments.append(dict(comment_item)) + + +def _patch_factory(fake): + """把 DouyinStoreFactory.create_store 替换为返回 fake,返回原方法用于还原。""" + orig = ds.DouyinStoreFactory.create_store + ds.DouyinStoreFactory.create_store = staticmethod(lambda: fake) + return orig + + +def _assert_no_forbidden(captured: dict, label: str): + hit = set(captured.keys()) & FORBIDDEN_KEYS + assert not hit, f"[{label}] 存储 dict 仍含禁用键: {hit}" + + +def _assert_raw_values_absent(captured: dict, raw_values, label: str): + """禁用的原始敏感值(uid/sec_uid/头像/签名/IP 等)不得出现在任何存储值里。 + creator_hash 是由 uid 派生的匿名哈希(已单独断言 ≠ 原文),不参与子串扫描。""" + leaked = [] + for rv in raw_values: + if not rv: + continue + for k, v in captured.items(): + if k == "creator_hash": + continue + if isinstance(v, str) and rv in v: + leaked.append((k, rv)) + assert not leaked, f"[{label}] 存储 dict 中泄漏了原始敏感值: {leaked}" + + +# ----------------------------- 测试 ----------------------------- + +def test_douyin_aweme_masks_user_info(): + aweme = _build_aweme_item() + raw_uid = aweme["author"]["uid"] + raw_nick = aweme["author"]["nickname"] + raw_sensitive = [ + raw_uid, + aweme["author"]["sec_uid"], + aweme["author"]["short_id"], + aweme["author"]["unique_id"], + aweme["author"]["avatar_thumb"]["url_list"][0], + aweme["author"]["signature"], + aweme["ip_label"], + ] + + fake = _FakeStore() + orig = _patch_factory(fake) + try: + asyncio.run(ds.update_douyin_aweme(aweme)) + finally: + ds.DouyinStoreFactory.create_store = orig + + assert len(fake.contents) == 1 + captured = fake.contents[0] + + # 1. 禁用键不出现 + _assert_no_forbidden(captured, "douyin_aweme") + # 2. 原始敏感值不泄漏到任何存储值 + _assert_raw_values_absent(captured, raw_sensitive, "douyin_aweme") + # 3. creator_hash 存在、≠原 uid、与 anonymize_user_id 一致 + assert captured.get("creator_hash") + assert captured["creator_hash"] != raw_uid + assert captured["creator_hash"] == anonymize_user_id(raw_uid) + # 4. nickname 保留但脱敏,≠原文,与 mask_nickname 一致 + assert captured.get("nickname") + assert captured["nickname"] != raw_nick + assert captured["nickname"] == mask_nickname(raw_nick) + # 5. 内容字段保留(desc/title 是作品描述,不禁用) + assert captured.get("desc") == aweme["desc"] + assert captured.get("title") == aweme["desc"] + # 6. _extract_* 内容字段正常提取(非空) + assert captured.get("cover_url") == "http://x/cover.jpg" + assert captured.get("video_download_url") == "http://x/video_h264.mp4" + assert captured.get("music_download_url") == "http://x/music.mp3" + assert "http://x/note_img1.jpg" in captured.get("note_download_url", "") + # 7. 互动数据拍平 + assert captured.get("liked_count") == "100" + assert captured.get("collected_count") == "5" + assert captured.get("comment_count") == "20" + assert captured.get("share_count") == "3" + assert captured.get("aweme_url") == f"https://www.douyin.com/video/{aweme['aweme_id']}" + + +def test_douyin_comment_masks_user_info(): + aweme_id = "7234567890123456" + comment = _build_comment_item() + raw_uid = comment["user"]["uid"] + raw_nick = comment["user"]["nickname"] + raw_sensitive = [ + raw_uid, + comment["user"]["sec_uid"], + comment["user"]["short_id"], + comment["user"]["unique_id"], + comment["user"]["avatar_medium"]["url_list"][0], + comment["user"]["signature"], + comment["ip_label"], + ] + + fake = _FakeStore() + orig = _patch_factory(fake) + try: + asyncio.run(ds.update_dy_aweme_comment(aweme_id, comment)) + finally: + ds.DouyinStoreFactory.create_store = orig + + assert len(fake.comments) == 1 + captured = fake.comments[0] + + _assert_no_forbidden(captured, "douyin_comment") + _assert_raw_values_absent(captured, raw_sensitive, "douyin_comment") + + assert captured.get("creator_hash") + assert captured["creator_hash"] != raw_uid + assert captured["creator_hash"] == anonymize_user_id(raw_uid) + assert captured.get("nickname") + assert captured["nickname"] != raw_nick + assert captured["nickname"] == mask_nickname(raw_nick) + + # 评论内容/ID 保留 + assert captured.get("content") == comment["text"] + assert captured.get("comment_id") == comment["cid"] + assert captured.get("aweme_id") == aweme_id + assert captured.get("parent_comment_id") == "0" + assert captured.get("sub_comment_count") == "2" + # 评论图片提取 + assert captured.get("pictures") == "http://x/cimg.jpg" + + +def test_douyin_store_end_to_end_sqlite(monkeypatch): + aweme = _build_aweme_item() + raw_uid = aweme["author"]["uid"] + raw_nick = aweme["author"]["nickname"] + + # ---- 1. FakeStore 捕获 update_douyin_aweme 产出的真实 dict ---- + fake = _FakeStore() + orig = _patch_factory(fake) + try: + asyncio.run(ds.update_douyin_aweme(aweme)) + finally: + ds.DouyinStoreFactory.create_store = orig + assert len(fake.contents) == 1 + captured = fake.contents[0] + + # ---- 2. DouyinAweme(**captured) 构造校验 ---- + # dict 多了已删列会 TypeError,少了非空必填列 SQLAlchemy 也会报错; + # 此处证明 captured 的 key 与删列后 ORM 列完全对得上,不抛异常。 + obj = DouyinAweme(**captured) + assert int(obj.aweme_id) == int(aweme["aweme_id"]) + assert obj.creator_hash == anonymize_user_id(raw_uid) + assert obj.creator_hash != raw_uid + assert obj.nickname == mask_nickname(raw_nick) + assert obj.nickname != raw_nick + assert obj.title == aweme["desc"] + assert obj.desc == aweme["desc"] + _assert_no_forbidden(captured, "douyin_aweme_orm_construct") + + # ---- 3. 端到端:内存 SQLite + 真实 store_content ---- + # 用 StaticPool 保证 :memory: 库在同一个连接上持久(跨 session 可见)。 + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + poolclass=StaticPool, + ) + # 让 db_session 用内存 engine;SAVE_DATA_OPTION=db 让工厂走 DouyinDbStoreImplement + monkeypatch.setattr(db_session, "get_async_engine", lambda *a, **kw: engine) + monkeypatch.setattr(config, "SAVE_DATA_OPTION", "db") + + async def _scenario(): + # 建表 + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + # 真实 store_content 路径(含 int(aweme_id) 与 if content_item.get("title") 判断) + await ds.update_douyin_aweme(aweme) + # 查询回读 + async with db_session.get_session() as session: + res = await session.execute( + select(DouyinAweme).where(DouyinAweme.aweme_id == int(aweme["aweme_id"])) + ) + row = res.scalar_one_or_none() + await engine.dispose() + return row + + row = asyncio.run(_scenario()) + + # ---- 4. 回读断言 ---- + assert row is not None, "作品未写入 SQLite" + assert int(row.aweme_id) == int(aweme["aweme_id"]) + assert row.creator_hash == anonymize_user_id(raw_uid) + assert row.creator_hash != raw_uid + assert row.nickname == mask_nickname(raw_nick) + assert row.nickname != raw_nick + assert row.desc == aweme["desc"] + assert row.title == aweme["desc"] + assert row.cover_url == "http://x/cover.jpg" + assert row.video_download_url == "http://x/video_h264.mp4" + assert row.music_download_url == "http://x/music.mp3" + # 禁用列在 ORM 上不存在(自省) + orm_cols = {c.name for c in DouyinAweme.__table__.columns} + assert not (orm_cols & FORBIDDEN_KEYS), f"ORM 仍含禁用列: {orm_cols & FORBIDDEN_KEYS}" + # captured 的所有 key 都是合法 ORM 列(无悬空 key) + assert set(captured.keys()).issubset(orm_cols), ( + f"captured 含非 ORM 列: {set(captured.keys()) - orm_cols}" + ) + + # 评论同样做一次 ORM 构造校验(证明 comment dict key 对得上) + comment = _build_comment_item() + fake_c = _FakeStore() + orig_c = _patch_factory(fake_c) + try: + asyncio.run(ds.update_dy_aweme_comment(comment["aweme_id"], comment)) + finally: + ds.DouyinStoreFactory.create_store = orig_c + captured_comment = fake_c.comments[0] + comment_obj = DouyinAwemeComment(**captured_comment) # 不抛异常即对得上 + assert int(comment_obj.comment_id) == int(comment["cid"]) + assert comment_obj.creator_hash == anonymize_user_id(comment["user"]["uid"]) + assert comment_obj.nickname == mask_nickname(comment["user"]["nickname"]) + _assert_no_forbidden(captured_comment, "douyin_comment_orm_construct") + comment_orm_cols = {c.name for c in DouyinAwemeComment.__table__.columns} + assert set(captured_comment.keys()).issubset(comment_orm_cols), ( + f"captured_comment 含非 ORM 列: {set(captured_comment.keys()) - comment_orm_cols}" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_kuaishou_no_user_info.py b/tests/test_kuaishou_no_user_info.py new file mode 100644 index 0000000..9061251 --- /dev/null +++ b/tests/test_kuaishou_no_user_info.py @@ -0,0 +1,284 @@ +# -*- coding: utf-8 -*- +""" +教学版回归测试：快手(kuaishou)存储链路不再持久化可定位真人的用户个人信息。 + +覆盖： +1. update_kuaishou_video —— mock video_item(含 author.id/name/headerUrl + photo 内容字段 + type) + 经 FakeStore 捕获，断言捕获 dict 不含禁用键(user_id/avatar/signature/ip_location/gender)、 + 含 creator_hash(≠原 user_id)、nickname 脱敏(≠原文)。 +2. update_ks_video_comment —— V2(snake_case) 与旧 GraphQL(camelCase) 两种 comment_item 格式各测一次。 +3. test_kuaishou_store_end_to_end_sqlite —— 端到端：FakeStore 捕获真实 dict -> + KuaishouVideo(**captured) 触发 ORM 列校验 -> 写入内存 SQLite -> 查询回读校验属性。 + +约束：只新建本测试文件，只读源码不改源码；不依赖网络/登录。 +""" +import asyncio +import contextlib +import types + +import pytest + +import store.kuaishou as ks +from store.kuaishou import update_kuaishou_video, update_ks_video_comment +from tools.user_hash import anonymize_user_id, mask_nickname + +# 教学版禁用字段(键)：一律不得出现在存储 dict 中。 +# 昵称字段 nickname 允许保留，但值须脱敏。 +FORBIDDEN_KEYS = {"user_id", "avatar", "signature", "ip_location", "gender"} + +# ----------------------------- mock 数据 ----------------------------- + +MOCK_VIDEO_ID = "3xf8e9kq2b7w4" +MOCK_AUTHOR_ID = "ks_author_001" +MOCK_AUTHOR_NAME = "快手达人" +MOCK_CAPTION = "这是一条测试视频，教学版脱敏回归 #测试" + +# 评论作者(两种格式用不同 id/昵称，便于分别断言) +MOCK_COMMENT_V2_AUTHOR_ID = "ks_user_888" +MOCK_COMMENT_V2_AUTHOR_NAME = "快手老铁" +MOCK_COMMENT_LEGACY_AUTHOR_ID = "ks_user_777" +MOCK_COMMENT_LEGACY_AUTHOR_NAME = "快乐源泉" + + +def make_mock_video() -> dict: + """贴近真实快手结构的 video_item：author 在顶层，photo 含内容字段。 + author.headerUrl 与各禁用字段一样不应进入存储 dict。""" + return { + "type": 1, + "photo": { + "id": MOCK_VIDEO_ID, + "caption": MOCK_CAPTION, + "timestamp": 1700000000000, + "coverUrl": "https://p.kuaishou.com/cover/abc.jpg", + "photoUrl": "https://v.kuaishou.com/play/abc.mp4", + "realLikeCount": 12345, + "viewCount": 67890, + }, + "author": { + "id": MOCK_AUTHOR_ID, + "name": MOCK_AUTHOR_NAME, + "headerUrl": "https://p.kuaishou.com/header/u001.jpg", + }, + } + + +def make_mock_comment_v2() -> dict: + """V2 API 格式：snake_case 字段名，comment_id 为 int。""" + return { + "comment_id": 9001, + "timestamp": 1700000001234, + "content": "太搞笑了哈哈哈", + "author_id": MOCK_COMMENT_V2_AUTHOR_ID, + "author_name": MOCK_COMMENT_V2_AUTHOR_NAME, + "headurl": "https://p.kuaishou.com/header/u888.jpg", + "commentCount": 7, + } + + +def make_mock_comment_legacy() -> dict: + """旧 GraphQL API 格式：camelCase 字段名。""" + return { + "commentId": 8001, + "timestamp": 1700000005678, + "content": "这条评论来自旧 GraphQL 接口", + "authorId": MOCK_COMMENT_LEGACY_AUTHOR_ID, + "authorName": MOCK_COMMENT_LEGACY_AUTHOR_NAME, + "headurl": "https://p.kuaishou.com/header/u777.jpg", + "subCommentCount": 3, + } + + +# ----------------------------- FakeStore 捕获 ----------------------------- + + +@contextlib.contextmanager +def _patch_create_store(): + """把 KuaishouStoreFactory.create_store 替换为返回捕获用 FakeStore 的 staticmethod。 + FakeStore 把 store_content/store_comment 收到的 dict 原样写入 holder.content / holder.comment。 + + 保存/还原走类 __dict__ 中的 staticmethod 描述符本身，避免把 staticmethod 退化为普通方法 + 而污染后续测试。""" + holder = types.SimpleNamespace(content={}, comment={}) + + class FakeStore: + async def store_content(self, content_item): + holder.content.clear() + holder.content.update(content_item) + + async def store_comment(self, comment_item): + holder.comment.clear() + holder.comment.update(comment_item) + + async def store_creator(self, creator): + pass + + orig = ks.KuaishouStoreFactory.__dict__["create_store"] + ks.KuaishouStoreFactory.create_store = staticmethod(lambda: FakeStore()) + try: + yield holder + finally: + ks.KuaishouStoreFactory.create_store = orig + + +def _assert_no_forbidden_keys(d: dict, label: str): + hit = set(d.keys()) & FORBIDDEN_KEYS + assert not hit, f"[{label}] 输出仍含禁用字段键: {hit}" + + +# ----------------------------- 测试 ----------------------------- + + +def test_kuaishou_video_masks_user_info(): + """video 链路：原始 user_id 转 creator_hash、昵称脱敏、headerUrl/禁用键不落库。""" + with _patch_create_store() as holder: + asyncio.run(update_kuaishou_video(make_mock_video())) + captured = holder.content + + assert captured, "FakeStore 未捕获到 content_item" + _assert_no_forbidden_keys(captured, "kuaishou_video") + # 头像字段不应进入存储 dict(author.headerUrl 已被丢弃) + assert "headerUrl" not in captured and "avatar" not in captured + + # creator_hash 存在且不等于原始 user_id + assert captured.get("creator_hash") == anonymize_user_id(MOCK_AUTHOR_ID) + assert captured["creator_hash"] != MOCK_AUTHOR_ID + assert captured["creator_hash"] # 非空 + + # 昵称已脱敏：等于 mask_nickname(原文) 且不等于原文 + assert captured.get("nickname") == mask_nickname(MOCK_AUTHOR_NAME) + assert captured["nickname"] != MOCK_AUTHOR_NAME + assert "*" in captured["nickname"] + + # 内容字段保留(video_id / desc / title) + assert captured["video_id"] == MOCK_VIDEO_ID + assert captured["desc"] == MOCK_CAPTION + assert captured["title"] == MOCK_CAPTION + # video_type 来自 video_item.type，被 str() 化 + assert captured["video_type"] == "1" + # 计数字段被 str() 化 + assert captured["liked_count"] == "12345" + assert captured["viewd_count"] == "67890" + + +def test_kuaishou_comment_v2_masks_user_info(): + """评论 V2(snake_case)格式：author_id/author_name 经匿名+脱敏，headurl 不落库。""" + with _patch_create_store() as holder: + asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_v2())) + captured = holder.comment + + assert captured, "FakeStore 未捕获到 comment_item" + _assert_no_forbidden_keys(captured, "kuaishou_comment_v2") + assert "headurl" not in captured and "avatar" not in captured + + # comment_id 由 int 转为 str + assert captured["comment_id"] == "9001" + assert captured["video_id"] == MOCK_VIDEO_ID + assert captured["content"] == "太搞笑了哈哈哈" + # V2 用 commentCount + assert captured["sub_comment_count"] == "7" + + # creator_hash / 昵称 + assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_V2_AUTHOR_ID) + assert captured["creator_hash"] != MOCK_COMMENT_V2_AUTHOR_ID + assert captured["nickname"] == mask_nickname(MOCK_COMMENT_V2_AUTHOR_NAME) + assert captured["nickname"] != MOCK_COMMENT_V2_AUTHOR_NAME + assert "*" in captured["nickname"] + + +def test_kuaishou_comment_legacy_masks_user_info(): + """评论旧 GraphQL(camelCase)格式：authorId/authorName 经匿名+脱敏，headurl 不落库。""" + with _patch_create_store() as holder: + asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_legacy())) + captured = holder.comment + + assert captured, "FakeStore 未捕获到 comment_item" + _assert_no_forbidden_keys(captured, "kuaishou_comment_legacy") + assert "headurl" not in captured and "avatar" not in captured + + # commentId 由 int 转为 str + assert captured["comment_id"] == "8001" + assert captured["video_id"] == MOCK_VIDEO_ID + assert captured["content"] == "这条评论来自旧 GraphQL 接口" + # 旧格式用 subCommentCount + assert captured["sub_comment_count"] == "3" + + # creator_hash / 昵称 + assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_LEGACY_AUTHOR_ID) + assert captured["creator_hash"] != MOCK_COMMENT_LEGACY_AUTHOR_ID + assert captured["nickname"] == mask_nickname(MOCK_COMMENT_LEGACY_AUTHOR_NAME) + assert captured["nickname"] != MOCK_COMMENT_LEGACY_AUTHOR_NAME + assert "*" in captured["nickname"] + + +def test_kuaishou_store_end_to_end_sqlite(): + """端到端：FakeStore 捕获真实 dict -> KuaishouVideo(**captured) ORM 列校验 + -> 写入内存 SQLite -> 查询回读，验证属性正确且无禁用列。 + + 全程在同一个事件循环内完成(aiosqlite 连接绑定事件循环，跨 loop 会报错)， + 不 patch db_session.get_session，而是直接用自建内存 engine 走 ORM 写入/查询。""" + from sqlalchemy import select + from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + from sqlalchemy.orm import sessionmaker + from sqlalchemy.pool import StaticPool + + from database.models import Base, KuaishouVideo, KuaishouVideoComment + + async def run(): + # 内存 SQLite + StaticPool：单连接共享，保证 create_all 与后续读写同一库 + engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool) + SessionFactory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + async with engine.begin() as conn: + await conn.run_sync( + Base.metadata.create_all, + tables=[KuaishouVideo.__table__, KuaishouVideoComment.__table__], + ) + + # 1) 经 update_kuaishou_video 产生真实存储 dict(FakeStore 捕获) + with _patch_create_store() as holder: + await update_kuaishou_video(make_mock_video()) + captured = holder.content + assert captured, "FakeStore 未捕获到 content_item" + + # 2) ORM 列校验：captured 的所有 key 必须是 KuaishouVideo 的合法列， + # 否则 KuaishouVideo(**captured) 抛 TypeError(若有禁用/多余键即暴露源码 bug) + valid_cols = {c.name for c in KuaishouVideo.__table__.columns} + assert set(captured.keys()) <= valid_cols, ( + f"captured 含非合法列: {set(captured.keys()) - valid_cols}" + ) + obj = KuaishouVideo(**captured) # 不抛异常即通过列校验 + assert obj.video_id == MOCK_VIDEO_ID + assert obj.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID) + assert obj.creator_hash != MOCK_AUTHOR_ID + assert obj.nickname == mask_nickname(MOCK_AUTHOR_NAME) + assert obj.nickname != MOCK_AUTHOR_NAME + assert obj.desc == MOCK_CAPTION # 内容保留 + + # 3) 真实写库 + 查询回读 + async with SessionFactory() as session: + session.add(obj) + await session.commit() + + res = await session.execute( + select(KuaishouVideo).where(KuaishouVideo.video_id == MOCK_VIDEO_ID) + ) + row = res.scalar_one() + assert row is not None + assert row.video_id == MOCK_VIDEO_ID + assert row.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID) + assert row.creator_hash != MOCK_AUTHOR_ID + assert row.nickname == mask_nickname(MOCK_AUTHOR_NAME) + assert row.nickname != MOCK_AUTHOR_NAME + # 内容字段保留 + assert row.desc == MOCK_CAPTION + assert row.title == MOCK_CAPTION + # ORM 行对象上不应存在任何禁用列属性 + for bad in FORBIDDEN_KEYS: + assert not hasattr(row, bad), f"KuaishouVideo 行对象仍含禁用属性: {bad}" + + await engine.dispose() + + asyncio.run(run()) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_no_user_info.py b/tests/test_no_user_info.py new file mode 100644 index 0000000..4314a74 --- /dev/null +++ b/tests/test_no_user_info.py @@ -0,0 +1,239 @@ +# -*- coding: utf-8 -*- +""" +教学版回归测试：确保爬取/存储链路不再持久化可定位真人的用户个人信息。 + +覆盖： +1. ORM 自省 —— database.models 中无禁用列、creator 档案表已删除、内容/评论表含 creator_hash。 +2. 提取层 —— 用 mock API/HTML payload 喂各平台提取器，断言输出 dict 不含禁用字段、 + 不含原始 user_id、昵称已脱敏且不等于原文。 +3. 仓库 grep 断言 —— store/ 与 media_platform/ 不再把禁用字段作为存储 dict 的 key。 +""" +import re +import subprocess +import pathlib + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent + +# 统一的禁用字段名(键)。昵称字段(nickname/user_nickname/screen_name/name/user_name)允许保留(值需脱敏)。 +FORBIDDEN_KEYS = { + "user_id", "sec_uid", "short_user_id", "user_unique_id", "user_signature", + "avatar", "user_avatar", "face", "sign", "profile_url", "user_link", + "url_token", "user_url_token", "ip_location", "ip_address", "gender", "sex", + "up_id", "fan_id", "up_name", "fan_name", "up_avatar", "fan_avatar", + "up_sign", "fan_sign", "mid", +} +NICK_KEYS = {"nickname", "user_nickname", "screen_name", "name", "user_name"} +MASK_RE = re.compile(r"^.?\*{1,4}.?$") + + +# ----------------------------- ORM 自省 ----------------------------- + +def test_orm_has_no_forbidden_columns(): + import database.models as m + from sqlalchemy.orm import class_mapper + tables = [c for c in dir(m) if c[0].isupper() + and c not in ("Base", "Column", "Integer", "BigInteger", "String", "Text")] + bad = [] + for t in tables: + cols = {c.name for c in class_mapper(getattr(m, t)).columns} + hit = cols & FORBIDDEN_KEYS + if hit: + bad.append((t, sorted(hit))) + assert not bad, f"ORM 仍含禁用列: {bad}" + + +def test_creator_tables_removed(): + import database.models as m + removed = {"XhsCreator", "DyCreator", "WeiboCreator", "TiebaCreator", + "ZhihuCreator", "BilibiliUpInfo", "BilibiliContactInfo"} + for t in removed: + assert not hasattr(m, t), f"creator 档案表 {t} 仍存在" + + +def test_content_tables_have_creator_hash(): + import database.models as m + from sqlalchemy.orm import class_mapper + content_tables = ["XhsNote", "XhsNoteComment", "WeiboNote", "WeiboNoteComment", + "BilibiliVideo", "BilibiliVideoComment", "BilibiliUpDynamic", + "DouyinAweme", "DouyinAwemeComment", "KuaishouVideo", + "KuaishouVideoComment", "TiebaNote", "TiebaComment", + "ZhihuContent", "ZhihuComment"] + for t in content_tables: + cols = {c.name for c in class_mapper(getattr(m, t)).columns} + assert "creator_hash" in cols, f"{t} 缺少 creator_hash 列" + + +# ----------------------------- 提取层 mock ----------------------------- + +def _check_no_forbidden_keys(d: dict, label: str): + keys = set(d.keys()) + hit = keys & FORBIDDEN_KEYS + assert not hit, f"[{label}] 输出仍含禁用字段键: {hit}" + + +def _check_nickname_masked(d: dict, raw: str, label: str): + nick_keys = set(d.keys()) & NICK_KEYS + assert nick_keys, f"[{label}] 未保留任何昵称字段(应保留并脱敏)" + for k in nick_keys: + val = d[k] + if val in ("", None): + continue + assert val != raw, f"[{label}] {k} 未脱敏，仍为原文: {val}" + assert MASK_RE.match(val) or "*" in val, f"[{label}] {k} 未脱敏: {val}" + + +def test_mask_and_hash_tools(): + from tools.user_hash import anonymize_user_id, mask_nickname + h = anonymize_user_id("12345") + assert h and h != "12345" and re.fullmatch(r"[0-9a-f]{16}", h) + assert anonymize_user_id(None) == "" and anonymize_user_id("") == "" + # 昵称脱敏：首尾留1字、中间星号，且不等于原文 + assert mask_nickname("张三丰") != "张三丰" + assert "*" in mask_nickname("张三丰") + assert mask_nickname(None) == "" + assert mask_nickname("a") == "*" + + +def test_xhs_note_extraction_masks_user_info(): + import asyncio + import store.xhs as xs + note_item = { + "note_id": "abc", + "type": "normal", + "title": "t", + "desc": "d", + "time": 1, + "last_update_time": 0, + "user": {"user_id": "u123", "nickname": "小红同学", "avatar": "http://x/a.jpg"}, + "ip_location": "上海", + "interact_info": {"liked_count": "1", "collected_count": "0", + "comment_count": "0", "share_count": "0"}, + "image_list": [], "tag_list": [], "xsec_token": "tok", + } + captured = {} + + class FakeStore: + async def store_content(self, content_item): + captured.update(content_item) + + orig = xs.XhsStoreFactory.create_store + xs.XhsStoreFactory.create_store = staticmethod(lambda: FakeStore()) + try: + asyncio.run(xs.update_xhs_note(note_item)) + finally: + xs.XhsStoreFactory.create_store = orig + _check_no_forbidden_keys(captured, "xhs_note") + assert captured.get("creator_hash") != "u123" + _check_nickname_masked(captured, "小红同学", "xhs_note") + + +def test_tieba_note_extraction_masks_user_info(): + from media_platform.tieba.help import TieBaExtractor + api_data = { + "thread": {"id": 1, "title": "tt", "reply_num": 5}, + "first_floor": {"tid": 1, "author_id": 9, "time": 1700000000, "content": "c"}, + "forum": {"name": "test", "id": 1}, + "page": {"total_page": 1}, + "user_list": [{"id": 9, "name_show": "贴吧老哥", "name": "lg", "portrait": "p", "ip_address": "北京"}], + } + note = TieBaExtractor().extract_note_detail_from_api(api_data) + d = note.model_dump() + _check_no_forbidden_keys(d, "tieba_note") + assert d.get("creator_hash") # user_link 已转哈希 + _check_nickname_masked(d, "贴吧老哥", "tieba_note") + + +def test_tieba_comment_extraction_masks_user_info(): + from media_platform.tieba.help import TieBaExtractor + from model.m_baidu_tieba import TiebaNote + api_data = { + "forum": {"id": 1, "name": "test"}, + "post_list": [{"id": 7, "author_id": 9, "time": 1700000000, "content": "c", "sub_post_number": 0}], + "user_list": [{"id": 9, "name_show": "评论员", "name": "py", "portrait": "p", "ip_address": "上海"}], + } + note_detail = TiebaNote(note_id="1", title="t", note_url="u", tieba_name="test", tieba_link="l") + comments = TieBaExtractor().extract_tieba_note_parent_comments_from_api(api_data, note_detail) + assert comments + d = comments[0].model_dump() + _check_no_forbidden_keys(d, "tieba_comment") + _check_nickname_masked(d, "评论员", "tieba_comment") + + +def test_zhihu_comment_extraction_masks_user_info(): + from media_platform.zhihu.help import ZhihuExtractor + from model.m_zhihu import ZhihuContent + comments_raw = [{ + "type": "comment", "id": 1, "content": "c", "created_time": 1700000000, + "like_count": 1, "dislike_count": 0, "child_comment_count": 0, + "author": {"id": "z9", "name": "知乎答主", "url_token": "tok", "avatar_url": "http://x/a.jpg"}, + "comment_tag": [{"type": "ip_info", "text": "广东"}], + }] + page_content = ZhihuContent(content_id="c1", content_type="answer") + comments = ZhihuExtractor().extract_comments(page_content, comments_raw) + assert comments + d = comments[0].model_dump() if hasattr(comments[0], "model_dump") else vars(comments[0]) + _check_no_forbidden_keys(d, "zhihu_comment") + assert "creator_hash" in d and d["creator_hash"] + _check_nickname_masked(d, "知乎答主", "zhihu_comment") + + +def test_bilibili_video_dict_masks_user_info(): + # 直接测 store/bilibili/__init__.py 的拍平逻辑(不触发网络) + import asyncio + from store.bilibili import update_bilibili_video + video_item = { + "View": { + "aid": 100, "title": "t", "desc": "d", "pubdate": 1, + "owner": {"mid": 777, "name": "UP主大人", "face": "http://x/a.jpg"}, + "stat": {"like": 1, "view": 2}, + "pic": "http://x/cover.jpg", + } + } + # 拦截真实存储：替换工厂返回一个捕获 dict 的假 store + captured = {} + + class FakeStore: + async def store_content(self, content_item): + captured.update(content_item) + + import store.bilibili as bs + orig = bs.BiliStoreFactory.create_store + bs.BiliStoreFactory.create_store = staticmethod(lambda: FakeStore()) + try: + asyncio.get_event_loop().run_until_complete(update_bilibili_video(video_item)) \ + if False else asyncio.run(update_bilibili_video(video_item)) + finally: + bs.BiliStoreFactory.create_store = orig + _check_no_forbidden_keys(captured, "bili_video") + assert captured.get("creator_hash") != 777 + _check_nickname_masked(captured, "UP主大人", "bili_video") + + +# ----------------------------- 仓库 grep 断言 ----------------------------- + +def test_store_no_forbidden_dict_keys(): + # store/ 下不得把禁用字段作为存储 dict 的 key("field": value 形式) + out = subprocess.run( + ["grep", "-rnE", '"(' + "|".join(FORBIDDEN_KEYS) + r')"\s*:', str(ROOT / "store")], + capture_output=True, text=True, + ) + # 允许的例外：Mongo store_creator 里的 query={"user_id": ...} 已全部改为 pass，应为空 + assert out.stdout.strip() == "", f"store/ 仍写入禁用字段键:\n{out.stdout}" + + +def test_store_no_creator_orm_imports(): + # 已删除的 creator ORM 表(XhsCreator/DyCreator/...)不得再从 database.models 导入。 + # 注意:model/m_*.py 里的同名 pydantic 类是内存类型，允许保留。 + out = subprocess.run( + ["grep", "-rnE", + r"from database\.models import.*(XhsCreator|DyCreator|WeiboCreator|TiebaCreator|ZhihuCreator|BilibiliUpInfo|BilibiliContactInfo)", + str(ROOT / "store")], + capture_output=True, text=True, + ) + assert out.stdout.strip() == "", f"store/ 仍 import 已删除的 creator ORM 表:\n{out.stdout}" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_weibo_no_user_info.py b/tests/test_weibo_no_user_info.py new file mode 100644 index 0000000..f584fdf --- /dev/null +++ b/tests/test_weibo_no_user_info.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +""" +教学版回归测试(微博 weibo):确保微博存储链路不再持久化可定位真人的用户个人信息。 + +覆盖: +1. test_weibo_note_masks_user_info —— 用贴近真实微博结构的 mock note_item 喂 + store.weibo.update_weibo_note,用 FakeStore 捕获拍平后的存储 dict,断言: + - 不含任何禁用字段键(user_id/avatar/gender/profile_url/ip_location/desc ...) + - 含 creator_hash,且不等于原始 user id + - nickname 已脱敏且不等于原文 +2. test_weibo_comment_masks_user_info —— 同上,对 update_weibo_note_comment。 +3. test_weibo_store_end_to_end_sqlite —— 端到端:把 update_weibo_note / + update_weibo_note_comment 产生的真实 dict 用 SQLite 内存库走完整 ORM + 写入+查询。WeiboNote(**captured_dict) 会因 SQLAlchemy 声明式构造器对未知 + 关键字的校验,在 dict 含已删列时直接抛 TypeError —— 以此证明 dict 的 key + 与删列后的 ORM 完全对得上,且表中无禁用列、有 creator_hash。 + +说明:微博 note 的正文存于 content 字段,update_weibo_note 不产生 desc 键 +(WeiboNote ORM 亦无 desc 列),故不存在用户 description 被持久化的风险。 +""" +import asyncio + +import pytest + +# 原始(明文)测试数据 +RAW_USER_ID = 7654321 +RAW_NICKNAME = "微博达人" +RAW_COMMENT_USER_ID = 111222 +RAW_COMMENT_NICKNAME = "评论员小张" +NOTE_ID = 5123456789 +COMMENT_ID = 998877 +# 合法 RFC2822 时间串(weekday 与日期已对齐:2025-06-14 是周六) +RFC2822_TIME = "Sat Jun 14 12:00:00 +0800 2025" + +# 禁用字段名(键)。昵称字段允许保留,但值必须脱敏。 +FORBIDDEN_KEYS = { + "user_id", "sec_uid", "short_user_id", "user_unique_id", + "avatar", "user_avatar", "face", "sign", "profile_url", "user_link", + "ip_location", "ip_address", "gender", "sex", "desc", +} + + +# ----------------------------- mock 数据 ----------------------------- + +def make_mock_note() -> dict: + """贴近真实 m.weibo.cn 接口结构的 mock note_item(含嵌套 user 信息)。""" + return { + "mblog": { + "id": NOTE_ID, + "text": "今天天气不错 @好友 出去玩", + "created_at": RFC2822_TIME, + "attitudes_count": 10, + "comments_count": 2, + "reposts_count": 1, + "user": { + "id": RAW_USER_ID, + "screen_name": RAW_NICKNAME, + "avatar_hd": "https://wx avatar.example.com/7654321.jpg", + "gender": "f", + "profile_url": "https://m.weibo.cn/profile/7654321", + "description": "这是一个用户签名", + "ip_location": "上海", + "followers_count": 9999, + }, + } + } + + +def make_mock_comment() -> dict: + """贴近真实微博评论结构的 mock comment_item(含嵌套 user 信息)。""" + return { + "id": COMMENT_ID, + "text": "说得好 支持", + "created_at": RFC2822_TIME, + "total_number": 3, + "like_count": 5, + "rootid": "parent_abc", + "user": { + "id": RAW_COMMENT_USER_ID, + "screen_name": RAW_COMMENT_NICKNAME, + "avatar_hd": "https://wx avatar.example.com/111222.jpg", + "gender": "m", + "profile_url": "https://m.weibo.cn/profile/111222", + "description": "评论员签名", + "ip_location": "广东", + }, + } + + +# ----------------------------- FakeStore 捕获 ----------------------------- + +class _FakeStore: + """捕获 store_content / store_comment 收到的 dict,不触发任何真实存储。""" + + def __init__(self): + self.captured_content = {} + self.captured_comment = {} + + async def store_content(self, content_item): + self.captured_content.update(content_item) + + async def store_comment(self, comment_item): + self.captured_comment.update(comment_item) + + async def store_creator(self, creator): + pass + + +def _patch_factory(fake: "_FakeStore"): + """把 store.weibo.WeibostoreFactory.create_store 替换为返回 fake 的静态方法, + 返回 (module, orig) 便于 finally 还原。""" + import store.weibo as wb + orig = wb.WeibostoreFactory.create_store + wb.WeibostoreFactory.create_store = staticmethod(lambda: fake) + return wb, orig + + +def _restore(wb, orig): + wb.WeibostoreFactory.create_store = orig + + +# ----------------------------- 测试 ----------------------------- + +def test_weibo_note_masks_user_info(): + """note 拍平后的存储 dict 不含禁用键、creator_hash 不等于原始 user id、昵称已脱敏。""" + import store.weibo as wb + + fake = _FakeStore() + wb_, orig = _patch_factory(fake) + try: + asyncio.run(wb.update_weibo_note(make_mock_note())) + finally: + _restore(wb_, orig) + + captured = fake.captured_content + assert captured, "FakeStore 未捕获到 note dict" + + # 1. 不含任何禁用字段键 + hit = set(captured.keys()) & FORBIDDEN_KEYS + assert not hit, f"note 存储 dict 仍含禁用字段键: {hit}" + + # 2. creator_hash 存在、是 16 位 hex、不等于原始 user id + creator_hash = captured.get("creator_hash") + assert creator_hash, "note dict 缺少 creator_hash" + assert creator_hash != str(RAW_USER_ID) + assert creator_hash != RAW_USER_ID + assert len(creator_hash) == 16 + + # 3. 昵称已脱敏:不等于原文且含星号 + nickname = captured.get("nickname") + assert nickname, "note dict 缺少 nickname" + assert nickname != RAW_NICKNAME, "note 昵称未脱敏,仍为原文" + assert "*" in nickname, f"note 昵称未脱敏: {nickname}" + + # 4. 内容字段正确(正文存于 content,不是 desc) + assert "hello" not in captured # 确认没误存 + assert "今天天气不错" in captured["content"] + assert captured["note_id"] == NOTE_ID + assert captured["liked_count"] == "10" + assert captured["comments_count"] == "2" + assert captured["shared_count"] == "1" + + +def test_weibo_comment_masks_user_info(): + """comment 拍平后的存储 dict 不含禁用键、creator_hash 不等于原始 user id、昵称已脱敏。""" + import store.weibo as wb + + fake = _FakeStore() + wb_, orig = _patch_factory(fake) + try: + asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment())) + finally: + _restore(wb_, orig) + + captured = fake.captured_comment + assert captured, "FakeStore 未捕获到 comment dict" + + # 1. 不含任何禁用字段键 + hit = set(captured.keys()) & FORBIDDEN_KEYS + assert not hit, f"comment 存储字典仍含禁用字段键: {hit}" + + # 2. creator_hash 存在、不等于原始 user id + creator_hash = captured.get("creator_hash") + assert creator_hash, "comment dict 缺少 creator_hash" + assert creator_hash != str(RAW_COMMENT_USER_ID) + assert creator_hash != RAW_COMMENT_USER_ID + assert len(creator_hash) == 16 + + # 3. 昵称已脱敏 + nickname = captured.get("nickname") + assert nickname, "comment dict 缺少 nickname" + assert nickname != RAW_COMMENT_NICKNAME, "comment 昵称未脱敏,仍为原文" + assert "*" in nickname, f"comment 昵称未脱敏: {nickname}" + + # 4. 内容字段正确 + assert "说得好" in captured["content"] + assert captured["comment_id"] == str(COMMENT_ID) + assert captured["comment_like_count"] == "5" + assert captured["sub_comment_count"] == "3" + assert captured["parent_comment_id"] == "parent_abc" + + +def test_weibo_store_end_to_end_sqlite(): + """端到端:捕获 note/comment 的真实 dict,用 SQLite 内存库走完整 ORM 写入+查询。 + + 关键点:WeiboNote(**captured_dict) / WeiboNoteComment(**captured_dict) 会触发 + SQLAlchemy 声明式构造器的关键字校验——若 dict 含已删列(如 avatar/gender)会直接 + 抛 TypeError。此处不抛异常即证明 dict 的 key 与删列后的 ORM 列完全对得上。 + """ + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + import store.weibo as wb + from database.models import Base, WeiboNote, WeiboNoteComment + + # ---- 1. 用 FakeStore 捕获 update_weibo_note / update_weibo_note_comment 产生的真实 dict ---- + fake = _FakeStore() + wb_, orig = _patch_factory(fake) + try: + asyncio.run(wb.update_weibo_note(make_mock_note())) + asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment())) + finally: + _restore(wb_, orig) + + captured_note = dict(fake.captured_content) + captured_comment = dict(fake.captured_comment) + assert captured_note and captured_comment + + # ---- 2. SQLite 内存库,建 weibo 两张表 ---- + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine, tables=[WeiboNote.__table__, WeiboNoteComment.__table__]) + Session = sessionmaker(bind=engine) + session = Session() + try: + # ---- 3. note:构造 ORM 对象(dict 多了已删列会直接 TypeError)并写入 ---- + note_obj = WeiboNote(**captured_note) # 不抛异常 => key 与 ORM 列对得上 + session.add(note_obj) + session.commit() + + row = session.query(WeiboNote).one() + note_cols = {c.name for c in WeiboNote.__table__.columns} + # 表结构层面无禁用列 + assert not (note_cols & FORBIDDEN_KEYS), \ + f"WeiboNote 表仍含禁用列: {note_cols & FORBIDDEN_KEYS}" + # 行数据层面:creator_hash 正确、昵称脱敏、正文保留 + assert row.creator_hash and row.creator_hash != str(RAW_USER_ID) + assert row.nickname != RAW_NICKNAME and "*" in row.nickname + assert row.note_id == NOTE_ID + assert "今天天气不错" in row.content + # 确认没有 desc 列存任何用户描述 + assert "desc" not in note_cols + + # ---- 4. comment:同上。注意 store_comment 实现会把 comment_id/note_id/create_time + # 转成 int 后再构造 ORM,此处忠实复刻该转换以走通 BigInteger 列写入 ---- + cc = dict(captured_comment) + cc["comment_id"] = int(cc["comment_id"]) + cc["note_id"] = int(cc.get("note_id", 0) or 0) + cc["create_time"] = int(cc.get("create_time", 0) or 0) + comment_obj = WeiboNoteComment(**cc) # 不抛异常 => key 与 ORM 列对得上 + session.add(comment_obj) + session.commit() + + crow = session.query(WeiboNoteComment).one() + comment_cols = {c.name for c in WeiboNoteComment.__table__.columns} + assert not (comment_cols & FORBIDDEN_KEYS), \ + f"WeiboNoteComment 表仍含禁用列: {comment_cols & FORBIDDEN_KEYS}" + assert crow.creator_hash and crow.creator_hash != str(RAW_COMMENT_USER_ID) + assert crow.nickname != RAW_COMMENT_NICKNAME and "*" in crow.nickname + assert crow.comment_id == COMMENT_ID + assert crow.note_id == NOTE_ID + assert "说得好" in crow.content + finally: + session.close() + engine.dispose() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tools/user_hash.py b/tools/user_hash.py new file mode 100644 index 0000000..89b314b --- /dev/null +++ b/tools/user_hash.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# 本文件为 MediaCrawler 教学版的一部分。 +# 出于教学与防骚扰定位，爬取结果中不保留任何可定位到真人的用户个人信息 +# （用户 ID、IP 归属地、头像、主页链接、签名、性别等一律不采集； +# 昵称保留但做中间脱敏）。本模块提供匿名化与脱敏工具。 +import hashlib + + +def anonymize_user_id(user_id) -> str: + """把原始用户 ID 转成匿名哈希，用于内容/评论记录的创作者分组， + 不暴露真实身份。返回 sha256 截断 16 位的十六进制串。""" + if user_id is None: + return "" + s = str(user_id).strip() + if not s: + return "" + return hashlib.sha256(s.encode("utf-8")).hexdigest()[:16] + + +def mask_nickname(name) -> str: + """昵称中间脱敏：首尾各保留 1 字，中间替换为星号。 + - 长度 <= 1：返回 "*" + - 长度 == 2：首字 + "*" + - 长度 >= 3：首字 + "***" + 尾字 + 这样既保留教学分析所需的内容归属语义，又无法据昵称定位到真人。 + """ + if name is None: + return "" + s = str(name) + if len(s) <= 1: + return "*" + if len(s) == 2: + return s[0] + "*" + return s[0] + "***" + s[-1]