jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-07-24 05:10:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 教学版移除全平台用户个人信息采集与持久化

Browse Source 
- 用户 ID 转为匿名 creator_hash，昵称中间脱敏，IP/头像/主页/签名/性别不再采集
- 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台
- 删除 7 张 creator 档案 ORM 表，15 张内容/评论表新增 creator_hash 列
- B 站禁用粉丝/关注/联系人列表抓取
- 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试

测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed)
This commit is contained in:
程序员阿江(Relakkes)
2026-07-01 13:09:55 +08:00
parent 8b4d8fcffa
commit 9f4f8bf768
26 changed files with 1426 additions and 822 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
database/models.py
View File

@@ -15,6 +15,14 @@
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
#
# 教学版说明：为防止爬取到的用户个人信息被用于定位真人并私信骚扰，
# 本 ORM 不再持久化任何可识别用户的字段（用户 ID、IP 归属地、头像、
# 主页链接、签名、性别等一律不落库）。原始用户 ID 在提取层经
# tools.user_hash.anonymize_user_id 转为匿名 creator_hash 后写入，
# 仅用于"同一创作者"的内容分组；昵称保留但经 mask_nickname 中间脱敏。
# 创作者个人档案表（XhsCreator/DyCreator/WeiboCreator/TiebaCreator/
# ZhihuCreator/BilibiliUpInfo/BilibiliContactInfo）已整体移除。
from sqlalchemy import create_engine, Column, Integer, Text, String, BigInteger
from sqlalchemy.ext.declarative import declarative_base
@@ -27,9 +35,8 @@ class BilibiliVideo(Base):
id = Column(Integer, primary_key=True, comment='主键ID')
video_id = Column(BigInteger, nullable=False, index=True, unique=True, comment='视频ID')
video_url = Column(Text, nullable=False, comment='视频URL')
user_id = Column(BigInteger, index=True, comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
liked_count = Column(Integer, comment='点赞数')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
@@ -50,11 +57,8 @@ class BilibiliVideo(Base):
class BilibiliVideoComment(Base):
__tablename__ = 'bilibili_video_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
sex = Column(Text, comment='性别')
sign = Column(Text, comment='签名')
avatar = Column(Text, comment='头像')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
@@ -65,41 +69,12 @@ class BilibiliVideoComment(Base):
parent_comment_id = Column(String(255), comment='父评论ID')
like_count = Column(Text, default='0', comment='点赞数')
class BilibiliUpInfo(Base):
__tablename__ = 'bilibili_up_info'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(BigInteger, index=True, comment='用户ID')
nickname = Column(Text, comment='用户昵称')
sex = Column(Text, comment='性别')
sign = Column(Text, comment='签名')
avatar = Column(Text, comment='头像')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
total_fans = Column(Integer, comment='总粉丝数')
total_liked = Column(Integer, comment='总获赞数')
user_rank = Column(Integer, comment='用户等级')
is_official = Column(Integer, comment='是否官方认证')
class BilibiliContactInfo(Base):
__tablename__ = 'bilibili_contact_info'
id = Column(Integer, primary_key=True, comment='主键ID')
up_id = Column(BigInteger, index=True, comment='UP主ID')
fan_id = Column(BigInteger, index=True, comment='粉丝ID')
up_name = Column(Text, comment='UP主名称')
fan_name = Column(Text, comment='粉丝名称')
up_sign = Column(Text, comment='UP主签名')
fan_sign = Column(Text, comment='粉丝签名')
up_avatar = Column(Text, comment='UP主头像')
fan_avatar = Column(Text, comment='粉丝头像')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
class BilibiliUpDynamic(Base):
__tablename__ = 'bilibili_up_dynamic'
id = Column(Integer, primary_key=True, comment='主键ID')
dynamic_id = Column(BigInteger, index=True, comment='动态ID')
user_id = Column(String(255), comment='用户ID')
user_name = Column(Text, comment='用户名称')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_name = Column(Text, comment='用户名称(已脱敏)')
text = Column(Text, comment='动态内容')
type = Column(Text, comment='动态类型')
pub_ts = Column(BigInteger, comment='发布时间戳')
@@ -112,14 +87,8 @@ class BilibiliUpDynamic(Base):
class DouyinAweme(Base):
__tablename__ = 'douyin_aweme'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
sec_uid = Column(String(255), comment='安全用户ID')
short_user_id = Column(String(255), comment='短用户ID')
user_unique_id = Column(String(255), comment='用户唯一ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
user_signature = Column(Text, comment='用户签名')
ip_location = Column(Text, comment='IP地址位置')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
aweme_id = Column(BigInteger, index=True, comment='作品ID')
@@ -141,14 +110,8 @@ class DouyinAweme(Base):
class DouyinAwemeComment(Base):
__tablename__ = 'douyin_aweme_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
sec_uid = Column(String(255), comment='安全用户ID')
short_user_id = Column(String(255), comment='短用户ID')
user_unique_id = Column(String(255), comment='用户唯一ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
user_signature = Column(Text, comment='用户签名')
ip_location = Column(Text, comment='IP地址位置')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
@@ -160,28 +123,11 @@ class DouyinAwemeComment(Base):
like_count = Column(Text, default='0', comment='点赞数')
pictures = Column(Text, default='', comment='图片')
class DyCreator(Base):
__tablename__ = 'dy_creator'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
ip_location = Column(Text, comment='IP地址位置')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
desc = Column(Text, comment='描述')
gender = Column(Text, comment='性别')
follows = Column(Text, comment='关注数')
fans = Column(Text, comment='粉丝数')
interaction = Column(Text, comment='互动数')
videos_count = Column(String(255), comment='视频数量')
class KuaishouVideo(Base):
__tablename__ = 'kuaishou_video'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(64), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
video_id = Column(String(255), index=True, comment='视频ID')
@@ -199,9 +145,8 @@ class KuaishouVideo(Base):
class KuaishouVideoComment(Base):
__tablename__ = 'kuaishou_video_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(Text, comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
@@ -213,12 +158,8 @@ class KuaishouVideoComment(Base):
class WeiboNote(Base):
__tablename__ = 'weibo_note'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
gender = Column(Text, comment='性别')
profile_url = Column(Text, comment='个人主页URL')
ip_location = Column(Text, default='', comment='IP地址位置')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
note_id = Column(BigInteger, index=True, comment='笔记ID')
@@ -234,12 +175,8 @@ class WeiboNote(Base):
class WeiboNoteComment(Base):
__tablename__ = 'weibo_note_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
gender = Column(Text, comment='性别')
profile_url = Column(Text, comment='个人主页URL')
ip_location = Column(Text, default='', comment='IP地址位置')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(BigInteger, index=True, comment='评论ID')
@@ -251,44 +188,11 @@ class WeiboNoteComment(Base):
sub_comment_count = Column(Text, comment='子评论数')
parent_comment_id = Column(String(255), comment='父评论ID')
class WeiboCreator(Base):
__tablename__ = 'weibo_creator'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
ip_location = Column(Text, comment='IP地址位置')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
desc = Column(Text, comment='描述')
gender = Column(Text, comment='性别')
follows = Column(Text, comment='关注数')
fans = Column(Text, comment='粉丝数')
tag_list = Column(Text, comment='标签列表')
class XhsCreator(Base):
__tablename__ = 'xhs_creator'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
ip_location = Column(Text, comment='IP地址位置')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
desc = Column(Text, comment='描述')
gender = Column(Text, comment='性别')
follows = Column(Text, comment='关注数')
fans = Column(Text, comment='粉丝数')
interaction = Column(Text, comment='互动数')
tag_list = Column(Text, comment='标签列表')
class XhsNote(Base):
__tablename__ = 'xhs_note'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
ip_location = Column(Text, comment='IP地址位置')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
note_id = Column(String(255), index=True, comment='笔记ID')
@@ -311,10 +215,8 @@ class XhsNote(Base):
class XhsNoteComment(Base):
__tablename__ = 'xhs_note_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(255), comment='用户ID')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
ip_location = Column(Text, comment='IP地址位置')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(String(255), index=True, comment='评论ID')
@@ -334,15 +236,13 @@ class TiebaNote(Base):
desc = Column(Text, comment='笔记描述')
note_url = Column(Text, comment='笔记URL')
publish_time = Column(String(255), index=True, comment='发布时间')
user_link = Column(Text, default='', comment='用户链接')
user_nickname = Column(Text, default='', comment='用户昵称')
user_avatar = Column(Text, default='', comment='用户头像')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, default='', comment='用户昵称(已脱敏)')
tieba_id = Column(String(255), default='', comment='贴吧ID')
tieba_name = Column(Text, comment='贴吧名称')
tieba_link = Column(Text, comment='贴吧链接')
total_replay_num = Column(Integer, default=0, comment='总回复数')
total_replay_page = Column(Integer, default=0, comment='总回复页数')
ip_location = Column(Text, default='', comment='IP地址位置')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
source_keyword = Column(Text, default='', comment='来源关键词')
@@ -353,35 +253,18 @@ class TiebaComment(Base):
comment_id = Column(String(255), index=True, comment='评论ID')
parent_comment_id = Column(String(255), default='', comment='父评论ID')
content = Column(Text, comment='评论内容')
user_link = Column(Text, default='', comment='用户链接')
user_nickname = Column(Text, default='', comment='用户昵称')
user_avatar = Column(Text, default='', comment='用户头像')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, default='', comment='用户昵称(已脱敏)')
tieba_id = Column(String(255), default='', comment='贴吧ID')
tieba_name = Column(Text, comment='贴吧名称')
tieba_link = Column(Text, comment='贴吧链接')
publish_time = Column(String(255), index=True, comment='发布时间')
ip_location = Column(Text, default='', comment='IP地址位置')
sub_comment_count = Column(Integer, default=0, comment='子评论数')
note_id = Column(String(255), index=True, comment='笔记ID')
note_url = Column(Text, comment='笔记URL')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
class TiebaCreator(Base):
__tablename__ = 'tieba_creator'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(64), comment='用户ID')
user_name = Column(Text, comment='用户名')
nickname = Column(Text, comment='用户昵称')
avatar = Column(Text, comment='用户头像')
ip_location = Column(Text, comment='IP地址位置')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
gender = Column(Text, comment='性别')
follows = Column(Text, comment='关注数')
fans = Column(Text, comment='粉丝数')
registration_duration = Column(Text, comment='注册时长')
class ZhihuContent(Base):
__tablename__ = 'zhihu_content'
id = Column(Integer, primary_key=True, comment='主键ID')
@@ -397,19 +280,11 @@ class ZhihuContent(Base):
voteup_count = Column(Integer, default=0, comment='赞同数')
comment_count = Column(Integer, default=0, comment='评论数')
source_keyword = Column(Text, comment='来源关键词')
user_id = Column(String(255), comment='用户ID')
user_link = Column(Text, comment='用户链接')
user_nickname = Column(Text, comment='用户昵称')
user_avatar = Column(Text, comment='用户头像')
user_url_token = Column(Text, comment='用户URL Token')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
# persist-1<persist1@126.com>
# Reason: Fixed ORM model definition error, ensuring consistency with database table structure.
# Side effects: None
# Rollback strategy: Restore this line
class ZhihuComment(Base):
__tablename__ = 'zhihu_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
@@ -417,36 +292,12 @@ class ZhihuComment(Base):
parent_comment_id = Column(String(64), comment='父评论ID')
content = Column(Text, comment='评论内容')
publish_time = Column(String(32), index=True, comment='发布时间')
ip_location = Column(Text, comment='IP地址位置')
sub_comment_count = Column(Integer, default=0, comment='子评论数')
like_count = Column(Integer, default=0, comment='点赞数')
dislike_count = Column(Integer, default=0, comment='点踩数')
content_id = Column(String(64), index=True, comment='内容ID')
content_type = Column(Text, comment='内容类型')
user_id = Column(String(64), comment='用户ID')
user_link = Column(Text, comment='用户链接')
user_nickname = Column(Text, comment='用户昵称')
user_avatar = Column(Text, comment='用户头像')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
class ZhihuCreator(Base):
__tablename__ = 'zhihu_creator'
id = Column(Integer, primary_key=True, comment='主键ID')
user_id = Column(String(64), unique=True, index=True, comment='用户ID')
user_link = Column(Text, comment='用户链接')
user_nickname = Column(Text, comment='用户昵称')
user_avatar = Column(Text, comment='用户头像')
url_token = Column(Text, comment='URL Token')
gender = Column(Text, comment='性别')
ip_location = Column(Text, comment='IP地址位置')
follows = Column(Integer, default=0, comment='关注数')
fans = Column(Integer, default=0, comment='粉丝数')
anwser_count = Column(Integer, default=0, comment='回答数')
video_count = Column(Integer, default=0, comment='视频数')
question_count = Column(Integer, default=0, comment='问题数')
article_count = Column(Integer, default=0, comment='文章数')
column_count = Column(Integer, default=0, comment='专栏数')
get_voteup_count = Column(Integer, default=0, comment='获赞数')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')

8
media_platform/bilibili/core.py
View File

@@ -646,14 +646,14 @@ class BilibiliCrawler(AbstractCrawler):
"""
async with semaphore:
creator_unhandled_info: Dict = await self.bili_client.get_creator_info(creator_id)
# 教学版：仅保留动态所需的最少字段(内存临时用)，不持久化创作者个人资料。
creator_info: Dict = {
"id": creator_id,
"name": creator_unhandled_info.get("name"),
"sign": creator_unhandled_info.get("sign"),
"avatar": creator_unhandled_info.get("face"),
}
await self.get_fans(creator_info, semaphore)
await self.get_followings(creator_info, semaphore)
# 教学版：不再爬取粉丝/关注列表(其他用户的个人信息)，防骚扰。
# await self.get_fans(creator_info, semaphore)
# await self.get_followings(creator_info, semaphore)
await self.get_dynamics(creator_info, semaphore)
async def get_fans(self, creator_info: Dict, semaphore: asyncio.Semaphore):

84
media_platform/tieba/help.py
View File

@@ -30,6 +30,7 @@ from parsel import Selector
from constant import baidu_tieba as const
from model.m_baidu_tieba import TiebaComment, TiebaCreator, TiebaNote
from tools import utils
from tools.user_hash import anonymize_user_id, mask_nickname
GENDER_MALE = "sex_male"
GENDER_FEMALE = "sex_female"
@@ -143,9 +144,8 @@ class TieBaExtractor:
publish_time=utils.get_time_str_from_unix_time(
item.get("time") or item.get("create_time") or 0
),
user_link="",
user_nickname=user.get("show_nickname") or user.get("user_name") or "",
user_avatar=user.get("portrait") or user.get("portraith") or "",
creator_hash=anonymize_user_id(user.get("id") or user.get("portrait") or ""),
user_nickname=mask_nickname(user.get("show_nickname") or user.get("user_name") or ""),
tieba_name=tieba_name,
tieba_link=self._tieba_link_from_name(tieba_name),
total_replay_num=item.get("post_num") or 0,
@@ -177,14 +177,12 @@ class TieBaExtractor:
publish_time=utils.get_time_str_from_unix_time(
first_floor.get("time") or thread.get("create_time") or 0
),
user_link=self._api_user_link(author),
user_nickname=author.get("name_show") or author.get("name") or "",
user_avatar=self._api_user_avatar(author),
creator_hash=anonymize_user_id(self._api_user_link(author)),
user_nickname=mask_nickname(author.get("name_show") or author.get("name") or ""),
tieba_name=tieba_name,
tieba_link=self._tieba_link_from_name(tieba_name),
total_replay_num=thread.get("reply_num") or 0,
total_replay_page=page.get("total_page") or 0,
ip_location=author.get("ip_address") or "",
)
return note
@@ -210,13 +208,11 @@ class TieBaExtractor:
sub_comment_count=item.get("sub_post_number") or 0,
content=self._extract_api_content_text(item.get("content")),
note_url=note_detail.note_url,
user_link=self._api_user_link(user),
user_nickname=user.get("name_show") or user.get("name") or "",
user_avatar=self._api_user_avatar(user),
creator_hash=anonymize_user_id(self._api_user_link(user)),
user_nickname=mask_nickname(user.get("name_show") or user.get("name") or ""),
tieba_id=tieba_id,
tieba_name=tieba_name,
tieba_link=tieba_link,
ip_location=user.get("ip_address") or "",
publish_time=utils.get_time_str_from_unix_time(item.get("time") or 0),
note_id=note_detail.note_id,
)
@@ -230,20 +226,11 @@ class TieBaExtractor:
user = api_data.get("data", {}).get("user", {})
if not user:
raise ValueError(f"Creator API response does not contain user info: {api_data}")
gender_value = user.get("sex", user.get("gender", 0))
gender = "Unknown"
if gender_value == 1:
gender = "Male"
elif gender_value == 2:
gender = "Female"
# 教学版：创作者个人资料不再落库，仅保留匿名哈希与脱敏昵称作内存对象。
return TiebaCreator(
user_id=str(user.get("id", "")),
user_name=str(user.get("name", "")),
nickname=str(user.get("name_show") or user.get("name") or ""),
avatar=self._api_user_avatar(user),
gender=gender,
ip_location=str(user.get("ip_address", "")),
creator_hash=anonymize_user_id(str(user.get("id", ""))),
user_nickname=mask_nickname(str(user.get("name_show") or user.get("name") or "")),
follows=int(user.get("concern_num") or 0),
fans=int(user.get("fans_num") or 0),
registration_duration=str(user.get("tb_age", "")),
@@ -370,10 +357,10 @@ class TieBaExtractor:
post, f".//div[{extractor._class_contains('p_content')}]"
),
note_url=note_url,
user_nickname=extractor._selector_text(
creator_hash=anonymize_user_id(extractor._absolute_url(user_selector.xpath("./@href").get(default=""))),
user_nickname=mask_nickname(extractor._selector_text(
post, ".//a[contains(@href, '/home/main')][1]"
),
user_link=extractor._absolute_url(user_selector.xpath("./@href").get(default="")),
)),
tieba_name=extractor._selector_text(
post, f".//a[{extractor._class_contains('p_forum')}][1]"
),
@@ -451,8 +438,8 @@ class TieBaExtractor:
title=title,
desc=desc,
note_url=f"{const.TIEBA_URL}/p/{note_id}",
user_nickname=user_nickname,
user_link="",
creator_hash="",
user_nickname=mask_nickname(user_nickname),
tieba_name=tieba_name,
tieba_link=tieba_link,
publish_time=publish_time,
@@ -499,8 +486,8 @@ class TieBaExtractor:
post_selector, f".//div[{self._class_contains('threadlist_abs')}]"
),
note_url=const.TIEBA_URL + f"/p/{note_id}",
user_link=self._absolute_url(user_selector.xpath("./@href").get(default="")),
user_nickname=user_nickname,
creator_hash=anonymize_user_id(self._absolute_url(user_selector.xpath("./@href").get(default=""))),
user_nickname=mask_nickname(user_nickname),
tieba_name=tieba_name,
tieba_link=tieba_link,
total_replay_num=post_field_value.get("reply_num", 0),
@@ -548,18 +535,14 @@ class TieBaExtractor:
title=content_selector.xpath("//title/text()").get(default="").strip(),
desc=content_selector.xpath("//meta[@name='description']/@content").get(default="").strip(),
note_url=const.TIEBA_URL + f"/p/{note_id}",
user_link=self._absolute_url(author_link),
user_nickname=(
creator_hash=anonymize_user_id(self._absolute_url(author_link)),
user_nickname=mask_nickname(
self._selector_text(first_floor_selector, f".//a[{self._class_contains('p_author_name')}][1]")
or author_value.get("user_nickname")
or author_value.get("user_name", "")
),
user_avatar=first_floor_selector.xpath(
f".//a[{self._class_contains('p_author_face')}]//img/@src"
).get(default="").strip(),
tieba_name=tieba_name,
tieba_link=tieba_link,
ip_location=ip_location,
publish_time=publish_time,
total_replay_num=(
thread_num_infos[0].xpath("./text()").get(default="0").strip()
@@ -623,13 +606,11 @@ class TieBaExtractor:
sub_comment_count=comment_content_value.get("comment_num") or 0,
content=utils.extract_text_from_html(content_html),
note_url=const.TIEBA_URL + f"/p/{note_id}",
user_link=self._absolute_url(user_selector.xpath("./@href").get(default="")),
user_nickname=user_nickname,
user_avatar=user_avatar,
creator_hash=anonymize_user_id(self._absolute_url(user_selector.xpath("./@href").get(default=""))),
user_nickname=mask_nickname(user_nickname),
tieba_id=str(comment_content_value.get("forum_id", "")),
tieba_name=tieba_name,
tieba_link=tieba_link,
ip_location=ip_location,
publish_time=publish_time,
note_id=note_id,
)
@@ -662,9 +643,8 @@ class TieBaExtractor:
comment_ele.xpath(f".//span[{self._class_contains('lzl_content_main')}]").get(default=""))
comment = TiebaComment(
comment_id=str(comment_value.get("spid")), content=content,
user_link=self._absolute_url(comment_user_a_selector.xpath("./@href").get(default="")),
user_nickname=str(comment_value.get("showname") or ""),
user_avatar=comment_user_a_selector.xpath("./img/@src").get(default=""),
creator_hash=anonymize_user_id(self._absolute_url(comment_user_a_selector.xpath("./@href").get(default=""))),
user_nickname=mask_nickname(str(comment_value.get("showname") or "")),
publish_time=self._selector_text(comment_ele, f".//span[{self._class_contains('lzl_time')}]"),
parent_comment_id=parent_comment.comment_id,
note_id=parent_comment.note_id, note_url=parent_comment.note_url,
@@ -695,13 +675,13 @@ class TieBaExtractor:
if len(follow_fans_selector) == 2:
follows, fans = self.extract_follow_and_fans(follow_fans_selector)
user_content = userinfo_userdata_selector.get(default='')
return TiebaCreator(user_id=user_id, user_name=user_name,
nickname=selector.xpath(".//span[@class='userinfo_username ']/text()").get(
default='').strip(),
avatar=selector.xpath(".//div[@class='userinfo_left_head']//img/@src").get(
default='').strip(),
gender=self.extract_gender(user_content),
ip_location=self.extract_ip(user_content),
# 教学版：创作者个人资料不再落库，仅保留匿名哈希与脱敏昵称作内存对象。
return TiebaCreator(creator_hash=anonymize_user_id(user_id or user_link),
user_nickname=mask_nickname(
selector.xpath(".//span[@class='userinfo_username ']/text()").get(
default='').strip()
or user_name
),
follows=follows,
fans=fans,
registration_duration=self.extract_registration_duration(user_content)
@@ -860,8 +840,8 @@ def test_extract_tieba_note_sub_comments():
with open("test_data/note_sub_comments.html", "r", encoding="utf-8") as f:
content = f.read()
extractor = TieBaExtractor()
fake_parment_comment = TiebaComment(comment_id="123456", content="content", user_link="user_link",
user_nickname="user_nickname", user_avatar="user_avatar",
fake_parment_comment = TiebaComment(comment_id="123456", content="content", creator_hash="creator_hash",
user_nickname="user_nickname",
publish_time="publish_time", parent_comment_id="parent_comment_id",
note_id="note_id", note_url="note_url", tieba_id="tieba_id",
tieba_name="tieba_name", )

20
media_platform/zhihu/client.py
View File

@@ -459,11 +459,11 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
}
return await self.get(uri, params)
async def get_all_anwser_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
async def get_all_anwser_by_creator(self, url_token: str, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
"""
Get all answers by creator
Args:
creator: Creator information
url_token: Creator url token (in-memory only, not persisted)
crawl_interval: Crawl delay interval in seconds
callback: Callback after completing one crawl
@@ -475,10 +475,10 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
offset: int = 0
limit: int = 20
while not is_end:
res = await self.get_creator_answers(creator.url_token, offset, limit)
res = await self.get_creator_answers(url_token, offset, limit)
if not res:
break
utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {creator.url_token} answers: {res}")
utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {url_token} answers: {res}")
paging_info = res.get("paging", {})
is_end = paging_info.get("is_end")
contents = self._extractor.extract_content_list_from_creator(res.get("data"))
@@ -491,14 +491,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_all_articles_by_creator(
self,
creator: ZhihuCreator,
url_token: str,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
"""
Get all articles by creator
Args:
creator:
url_token: Creator url token (in-memory only, not persisted)
crawl_interval:
callback:
@@ -510,7 +510,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
offset: int = 0
limit: int = 20
while not is_end:
res = await self.get_creator_articles(creator.url_token, offset, limit)
res = await self.get_creator_articles(url_token, offset, limit)
if not res:
break
paging_info = res.get("paging", {})
@@ -525,14 +525,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_all_videos_by_creator(
self,
creator: ZhihuCreator,
url_token: str,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
"""
Get all videos by creator
Args:
creator:
url_token: Creator url token (in-memory only, not persisted)
crawl_interval:
callback:
@@ -544,7 +544,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
offset: int = 0
limit: int = 20
while not is_end:
res = await self.get_creator_videos(creator.url_token, offset, limit)
res = await self.get_creator_videos(url_token, offset, limit)
if not res:
break
paging_info = res.get("paging", {})

7
media_platform/zhihu/core.py
View File

@@ -276,27 +276,26 @@ class ZhihuCrawler(AbstractCrawler):
utils.logger.info(
f"[ZhihuCrawler.get_creators_and_notes] Creator info: {createor_info}"
)
await zhihu_store.save_creator(creator=createor_info)
# By default, only answer information is extracted, uncomment below if articles and videos are needed
# Get all anwser information of the creator
all_content_list = await self.zhihu_client.get_all_anwser_by_creator(
creator=createor_info,
url_token=user_url_token,
crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
callback=zhihu_store.batch_update_zhihu_contents,
)
# Get all articles of the creator's contents
# all_content_list = await self.zhihu_client.get_all_articles_by_creator(
# creator=createor_info,
# url_token=user_url_token,
# crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
# callback=zhihu_store.batch_update_zhihu_contents
# )
# Get all videos of the creator's contents
# all_content_list = await self.zhihu_client.get_all_videos_by_creator(
# creator=createor_info,
# url_token=user_url_token,
# crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
# callback=zhihu_store.batch_update_zhihu_contents
# )

37
media_platform/zhihu/help.py
View File

@@ -30,6 +30,7 @@ from constant import zhihu as zhihu_constant
from model.m_zhihu import ZhihuComment, ZhihuContent, ZhihuCreator
from tools import utils
from tools.crawler_util import extract_text_from_html
from tools.user_hash import anonymize_user_id, mask_nickname
ZHIHU_SGIN_JS = None
@@ -120,11 +121,8 @@ class ZhihuExtractor:
# extract author info
author_info = self._extract_content_or_comment_author(answer.get("author"))
res.user_id = author_info.user_id
res.user_link = author_info.user_link
res.creator_hash = author_info.creator_hash
res.user_nickname = author_info.user_nickname
res.user_avatar = author_info.user_avatar
res.user_url_token = author_info.url_token
return res
def _extract_article_content(self, article: Dict) -> ZhihuContent:
@@ -150,11 +148,8 @@ class ZhihuExtractor:
# extract author info
author_info = self._extract_content_or_comment_author(article.get("author"))
res.user_id = author_info.user_id
res.user_link = author_info.user_link
res.creator_hash = author_info.creator_hash
res.user_nickname = author_info.user_nickname
res.user_avatar = author_info.user_avatar
res.user_url_token = author_info.url_token
return res
def _extract_zvideo_content(self, zvideo: Dict) -> ZhihuContent:
@@ -184,11 +179,8 @@ class ZhihuExtractor:
# extract author info
author_info = self._extract_content_or_comment_author(zvideo.get("author"))
res.user_id = author_info.user_id
res.user_link = author_info.user_link
res.creator_hash = author_info.creator_hash
res.user_nickname = author_info.user_nickname
res.user_avatar = author_info.user_avatar
res.user_url_token = author_info.url_token
return res
@staticmethod
@@ -207,11 +199,8 @@ class ZhihuExtractor:
return res
if not author.get("id"):
author = author.get("member")
res.user_id = author.get("id")
res.user_link = f"{zhihu_constant.ZHIHU_URL}/people/{author.get('url_token')}"
res.user_nickname = author.get("name")
res.user_avatar = author.get("avatar_url")
res.url_token = author.get("url_token")
res.creator_hash = anonymize_user_id(author.get("id"))
res.user_nickname = mask_nickname(author.get("name"))
except Exception as e :
utils.logger.warning(
@@ -253,7 +242,6 @@ class ZhihuExtractor:
res.parent_comment_id = comment.get("reply_comment_id")
res.content = extract_text_from_html(comment.get("content"))
res.publish_time = comment.get("created_time")
res.ip_location = self._extract_comment_ip_location(comment.get("comment_tag", []))
res.sub_comment_count = comment.get("child_comment_count")
res.like_count = comment.get("like_count") if comment.get("like_count") else 0
res.dislike_count = comment.get("dislike_count") if comment.get("dislike_count") else 0
@@ -262,10 +250,8 @@ class ZhihuExtractor:
# extract author info
author_info = self._extract_content_or_comment_author(comment.get("author"))
res.user_id = author_info.user_id
res.user_link = author_info.user_link
res.creator_hash = author_info.creator_hash
res.user_nickname = author_info.user_nickname
res.user_avatar = author_info.user_avatar
return res
@staticmethod
@@ -352,13 +338,8 @@ class ZhihuExtractor:
return None
res = ZhihuCreator()
res.user_id = creator_info.get("id")
res.user_link = f"{zhihu_constant.ZHIHU_URL}/people/{user_url_token}"
res.user_nickname = creator_info.get("name")
res.user_avatar = creator_info.get("avatarUrl")
res.url_token = creator_info.get("urlToken") or user_url_token
res.gender = self._foramt_gender_text(creator_info.get("gender"))
res.ip_location = creator_info.get("ipInfo")
res.creator_hash = anonymize_user_id(creator_info.get("id"))
res.user_nickname = mask_nickname(creator_info.get("name"))
res.follows = creator_info.get("followingCount")
res.fans = creator_info.get("followerCount")
res.anwser_count = creator_info.get("answerCount")

22
model/m_baidu_tieba.py
View File

@@ -33,14 +33,12 @@ class TiebaNote(BaseModel):
desc: str = Field(default="", description="Post description")
note_url: str = Field(..., description="Post link")
publish_time: str = Field(default="", description="Publish time")
user_link: str = Field(default="", description="User homepage link")
user_nickname: str = Field(default="", description="User nickname")
user_avatar: str = Field(default="", description="User avatar URL")
creator_hash: str = Field(default="", description="创作者匿名哈希(不存原始用户链接)")
user_nickname: str = Field(default="", description="User nickname (已脱敏)")
tieba_name: str = Field(..., description="Tieba name")
tieba_link: str = Field(..., description="Tieba link")
total_replay_num: int = Field(default=0, description="Total reply count")
total_replay_page: int = Field(default=0, description="Total reply pages")
ip_location: Optional[str] = Field(default="", description="IP location")
source_keyword: str = Field(default="", description="Source keyword")
@@ -52,11 +50,9 @@ class TiebaComment(BaseModel):
comment_id: str = Field(..., description="Comment ID")
parent_comment_id: str = Field(default="", description="Parent comment ID")
content: str = Field(..., description="Comment content")
user_link: str = Field(default="", description="User homepage link")
user_nickname: str = Field(default="", description="User nickname")
user_avatar: str = Field(default="", description="User avatar URL")
creator_hash: str = Field(default="", description="创作者匿名哈希(不存原始用户链接)")
user_nickname: str = Field(default="", description="User nickname (已脱敏)")
publish_time: str = Field(default="", description="Publish time")
ip_location: Optional[str] = Field(default="", description="IP location")
sub_comment_count: int = Field(default=0, description="Sub-comment count")
note_id: str = Field(..., description="Post ID")
note_url: str = Field(..., description="Post link")
@@ -67,14 +63,10 @@ class TiebaComment(BaseModel):
class TiebaCreator(BaseModel):
"""
Baidu Tieba creator
Baidu Tieba creator（教学版：个人资料不再落库，仅作内存对象）
"""
user_id: str = Field(..., description="User ID")
user_name: str = Field(..., description="Username")
nickname: str = Field(..., description="User nickname")
gender: str = Field(default="", description="User gender")
avatar: str = Field(..., description="User avatar URL")
ip_location: Optional[str] = Field(default="", description="IP location")
creator_hash: str = Field(default="", description="创作者匿名哈希(不存原始用户链接)")
user_nickname: str = Field(default="", description="User nickname (已脱敏)")
follows: int = Field(default=0, description="Follows count")
fans: int = Field(default=0, description="Fans count")
registration_duration: str = Field(default="", description="Registration duration")

29
model/m_zhihu.py
View File

@@ -19,8 +19,6 @@
# -*- coding: utf-8 -*-
from typing import Optional
from pydantic import BaseModel, Field
@@ -40,12 +38,8 @@ class ZhihuContent(BaseModel):
voteup_count: int = Field(default=0, description="Upvote count")
comment_count: int = Field(default=0, description="Comment count")
source_keyword: str = Field(default="", description="Source keyword")
user_id: str = Field(default="", description="User ID")
user_link: str = Field(default="", description="User homepage link")
user_nickname: str = Field(default="", description="User nickname")
user_avatar: str = Field(default="", description="User avatar URL")
user_url_token: str = Field(default="", description="User url_token")
creator_hash: str = Field(default="", description="Creator anonymized hash")
user_nickname: str = Field(default="", description="User nickname (masked)")
class ZhihuComment(BaseModel):
@@ -57,30 +51,21 @@ class ZhihuComment(BaseModel):
parent_comment_id: str = Field(default="", description="Parent comment ID")
content: str = Field(default="", description="Comment content")
publish_time: int = Field(default=0, description="Publish time")
ip_location: Optional[str] = Field(default="", description="IP location")
sub_comment_count: int = Field(default=0, description="Sub-comment count")
like_count: int = Field(default=0, description="Like count")
dislike_count: int = Field(default=0, description="Dislike count")
content_id: str = Field(default="", description="Content ID")
content_type: str = Field(default="", description="Content type (article | answer | zvideo)")
user_id: str = Field(default="", description="User ID")
user_link: str = Field(default="", description="User homepage link")
user_nickname: str = Field(default="", description="User nickname")
user_avatar: str = Field(default="", description="User avatar URL")
creator_hash: str = Field(default="", description="Creator anonymized hash")
user_nickname: str = Field(default="", description="User nickname (masked)")
class ZhihuCreator(BaseModel):
"""
Zhihu creator
Zhihu creator (in-memory only; personal profile is no longer persisted)
"""
user_id: str = Field(default="", description="User ID")
user_link: str = Field(default="", description="User homepage link")
user_nickname: str = Field(default="", description="User nickname")
user_avatar: str = Field(default="", description="User avatar URL")
url_token: str = Field(default="", description="User url_token")
gender: str = Field(default="", description="User gender")
ip_location: Optional[str] = Field(default="", description="IP location")
creator_hash: str = Field(default="", description="Creator anonymized hash")
user_nickname: str = Field(default="", description="User nickname (masked)")
follows: int = Field(default=0, description="Follows count")
fans: int = Field(default=0, description="Fans count")
anwser_count: int = Field(default=0, description="Answer count")

70
store/bilibili/__init__.py
View File

@@ -26,6 +26,7 @@ from typing import List
import config
from var import source_keyword_var
from tools.user_hash import anonymize_user_id, mask_nickname
from ._store_impl import *
from .bilibilli_store_media import *
@@ -62,9 +63,8 @@ async def update_bilibili_video(video_item: Dict):
"title": video_item_view.get("title", "")[:500],
"desc": video_item_view.get("desc", "")[:500],
"create_time": video_item_view.get("pubdate"),
"user_id": str(video_user_info.get("mid")),
"nickname": video_user_info.get("name"),
"avatar": video_user_info.get("face", ""),
"creator_hash": anonymize_user_id(video_user_info.get("mid")), # 创作者匿名哈希(不存原始 mid)
"nickname": mask_nickname(video_user_info.get("name")), # 用户昵称(已脱敏)
"liked_count": str(video_item_stat.get("like", "")),
"disliked_count": str(video_item_stat.get("dislike", "")),
"video_play_count": str(video_item_stat.get("view", "")),
@@ -83,22 +83,8 @@ async def update_bilibili_video(video_item: Dict):
async def update_up_info(video_item: Dict):
video_item_card_list: Dict = video_item.get("Card")
video_item_card: Dict = video_item_card_list.get("card")
saver_up_info = {
"user_id": str(video_item_card.get("mid")),
"nickname": video_item_card.get("name"),
"sex": video_item_card.get("sex"),
"sign": video_item_card.get("sign"),
"avatar": video_item_card.get("face"),
"last_modify_ts": utils.get_current_timestamp(),
"total_fans": video_item_card.get("fans"),
"total_liked": video_item_card_list.get("like_num"),
"user_rank": video_item_card.get("level_info").get("current_level"),
"is_official": video_item_card.get("official_verify").get("type"),
}
utils.logger.info(f"[store.bilibili.update_up_info] bilibili user_id:{video_item_card.get('mid')}")
await BiliStoreFactory.create_store().store_creator(creator=saver_up_info)
# 教学版：UP 主个人资料(昵称/性别/签名/头像/粉丝数等)不再落库，防骚扰。
return
async def batch_update_bilibili_video_comments(video_id: str, comments: List[Dict]):
@@ -120,11 +106,8 @@ async def update_bilibili_video_comment(video_id: str, comment_item: Dict):
"create_time": comment_item.get("ctime"),
"video_id": str(video_id),
"content": content.get("message"),
"user_id": user_info.get("mid"),
"nickname": user_info.get("uname"),
"sex": user_info.get("sex"),
"sign": user_info.get("sign"),
"avatar": user_info.get("avatar"),
"creator_hash": anonymize_user_id(user_info.get("mid")), # 创作者匿名哈希(不存原始 mid)
"nickname": mask_nickname(user_info.get("uname")), # 用户昵称(已脱敏)
"sub_comment_count": str(comment_item.get("rcount", 0)),
"like_count": like_count,
"last_modify_ts": utils.get_current_timestamp(),
@@ -149,29 +132,13 @@ async def store_video(aid, video_content, extension_file_name):
async def batch_update_bilibili_creator_fans(creator_info: Dict, fans_list: List[Dict]):
if not fans_list:
# 教学版：不再采集/存储粉丝列表(其他用户的个人信息)，防骚扰。
return
for fan_item in fans_list:
fan_info: Dict = {
"id": fan_item.get("mid"),
"name": fan_item.get("uname"),
"sign": fan_item.get("sign"),
"avatar": fan_item.get("face"),
}
await update_bilibili_creator_contact(creator_info=creator_info, fan_info=fan_info)
async def batch_update_bilibili_creator_followings(creator_info: Dict, followings_list: List[Dict]):
if not followings_list:
# 教学版：不再采集/存储关注列表(其他用户的个人信息)，防骚扰。
return
for following_item in followings_list:
following_info: Dict = {
"id": following_item.get("mid"),
"name": following_item.get("uname"),
"sign": following_item.get("sign"),
"avatar": following_item.get("face"),
}
await update_bilibili_creator_contact(creator_info=following_info, fan_info=creator_info)
async def batch_update_bilibili_creator_dynamics(creator_info: Dict, dynamics_list: List[Dict]):
@@ -201,26 +168,15 @@ async def batch_update_bilibili_creator_dynamics(creator_info: Dict, dynamics_li
async def update_bilibili_creator_contact(creator_info: Dict, fan_info: Dict):
save_contact_item = {
"up_id": creator_info["id"],
"fan_id": fan_info["id"],
"up_name": creator_info["name"],
"fan_name": fan_info["name"],
"up_sign": creator_info["sign"],
"fan_sign": fan_info["sign"],
"up_avatar": creator_info["avatar"],
"fan_avatar": fan_info["avatar"],
"last_modify_ts": utils.get_current_timestamp(),
}
await BiliStoreFactory.create_store().store_contact(contact_item=save_contact_item)
# 教学版：UP-粉丝关系表已移除，不再存储联系人信息。
return
async def update_bilibili_creator_dynamic(creator_info: Dict, dynamic_info: Dict):
save_dynamic_item = {
"dynamic_id": dynamic_info["dynamic_id"],
"user_id": creator_info["id"],
"user_name": creator_info["name"],
"creator_hash": anonymize_user_id(creator_info.get("id")), # 创作者匿名哈希(不存原始 ID)
"user_name": mask_nickname(creator_info.get("name")), # 用户名称(已脱敏)
"text": dynamic_info["text"],
"type": dynamic_info["type"],
"pub_ts": dynamic_info["pub_ts"],

76
store/bilibili/_store_impl.py
View File

@@ -36,7 +36,7 @@ from sqlalchemy.orm import sessionmaker
import config
from base.base_crawler import AbstractStore
from database.db_session import get_session
from database.models import BilibiliVideoComment, BilibiliVideo, BilibiliUpInfo, BilibiliUpDynamic, BilibiliContactInfo
from database.models import BilibiliVideoComment, BilibiliVideo, BilibiliUpDynamic
from tools.async_file_writer import AsyncFileWriter
from tools import utils, words
from var import crawler_type_var
@@ -130,7 +130,6 @@ class BiliDbStoreImplement(AbstractStore):
"""
video_id = int(content_item.get("video_id"))
content_item["video_id"] = video_id
content_item["user_id"] = int(content_item.get("user_id", 0) or 0)
content_item["liked_count"] = int(content_item.get("liked_count", 0) or 0)
content_item["create_time"] = int(content_item.get("create_time", 0) or 0)
@@ -179,60 +178,12 @@ class BiliDbStoreImplement(AbstractStore):
await session.commit()
async def store_creator(self, creator: Dict):
"""
Bilibili creator DB storage implementation
Args:
creator: creator item dict
"""
creator_id = int(creator.get("user_id"))
creator["user_id"] = creator_id
creator["total_fans"] = int(creator.get("total_fans", 0) or 0)
creator["total_liked"] = int(creator.get("total_liked", 0) or 0)
creator["user_rank"] = int(creator.get("user_rank", 0) or 0)
creator["is_official"] = int(creator.get("is_official", 0) or 0)
async with get_session() as session:
result = await session.execute(select(BilibiliUpInfo).where(BilibiliUpInfo.user_id == creator_id))
creator_detail = result.scalar_one_or_none()
if not creator_detail:
creator["add_ts"] = utils.get_current_timestamp()
creator["last_modify_ts"] = utils.get_current_timestamp()
new_creator = BilibiliUpInfo(**creator)
session.add(new_creator)
else:
creator["last_modify_ts"] = utils.get_current_timestamp()
for key, value in creator.items():
setattr(creator_detail, key, value)
await session.commit()
# 教学版：UP 主个人资料不再落库
pass
async def store_contact(self, contact_item: Dict):
"""
Bilibili contact DB storage implementation
Args:
contact_item: contact item dict
"""
up_id = int(contact_item.get("up_id"))
fan_id = int(contact_item.get("fan_id"))
contact_item["up_id"] = up_id
contact_item["fan_id"] = fan_id
async with get_session() as session:
result = await session.execute(
select(BilibiliContactInfo).where(BilibiliContactInfo.up_id == up_id, BilibiliContactInfo.fan_id == fan_id)
)
contact_detail = result.scalar_one_or_none()
if not contact_detail:
contact_item["add_ts"] = utils.get_current_timestamp()
contact_item["last_modify_ts"] = utils.get_current_timestamp()
new_contact = BilibiliContactInfo(**contact_item)
session.add(new_contact)
else:
contact_item["last_modify_ts"] = utils.get_current_timestamp()
for key, value in contact_item.items():
setattr(contact_detail, key, value)
await session.commit()
# 教学版：UP-粉丝关系表已移除，不再存储联系人信息
pass
async def store_dynamic(self, dynamic_item):
"""
@@ -421,21 +372,8 @@ class BiliMongoStoreImplement(AbstractStore):
utils.logger.info(f"[BiliMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB")
async def store_creator(self, creator_item: Dict):
"""
Store UP master information to MongoDB
Args:
creator_item: UP master data
"""
user_id = creator_item.get("user_id")
if not user_id:
return
await self.mongo_store.save_or_update(
collection_suffix="creators",
query={"user_id": user_id},
data=creator_item
)
utils.logger.info(f"[BiliMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
# 教学版：UP 主个人资料不再落库
pass
class BiliExcelStoreImplement:

42
store/douyin/__init__.py
View File

@@ -25,6 +25,7 @@ from typing import List
import config
from var import source_keyword_var
from tools.user_hash import anonymize_user_id, mask_nickname
from ._store_impl import *
from .douyin_store_media import *
@@ -164,18 +165,12 @@ async def update_douyin_aweme(aweme_item: Dict):
"title": aweme_item.get("desc", ""),
"desc": aweme_item.get("desc", ""),
"create_time": aweme_item.get("create_time"),
"user_id": user_info.get("uid"),
"sec_uid": user_info.get("sec_uid"),
"short_user_id": user_info.get("short_id"),
"user_unique_id": user_info.get("unique_id"),
"user_signature": user_info.get("signature"),
"nickname": user_info.get("nickname"),
"avatar": user_info.get("avatar_thumb", {}).get("url_list", [""])[0],
"creator_hash": anonymize_user_id(user_info.get("uid")), # 创作者匿名哈希(不存原始 uid)
"nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏)
"liked_count": str(interact_info.get("digg_count")),
"collected_count": str(interact_info.get("collect_count")),
"comment_count": str(interact_info.get("comment_count")),
"share_count": str(interact_info.get("share_count")),
"ip_location": aweme_item.get("ip_label", ""),
"last_modify_ts": utils.get_current_timestamp(),
"aweme_url": f"https://www.douyin.com/video/{aweme_id}",
"cover_url": _extract_content_cover_url(aweme_item),
@@ -203,20 +198,13 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
user_info = comment_item.get("user", {})
comment_id = comment_item.get("cid")
parent_comment_id = comment_item.get("reply_id", "0")
avatar_info = (user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get("avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {})
save_comment_item = {
"comment_id": comment_id,
"create_time": comment_item.get("create_time"),
"ip_location": comment_item.get("ip_label", ""),
"aweme_id": aweme_id,
"content": comment_item.get("text"),
"user_id": user_info.get("uid"),
"sec_uid": user_info.get("sec_uid"),
"short_user_id": user_info.get("short_id"),
"user_unique_id": user_info.get("unique_id"),
"user_signature": user_info.get("signature"),
"nickname": user_info.get("nickname"),
"avatar": avatar_info.get("url_list", [""])[0],
"creator_hash": anonymize_user_id(user_info.get("uid")), # 创作者匿名哈希(不存原始 uid)
"nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏)
"sub_comment_count": str(comment_item.get("reply_comment_total", 0)),
"like_count": (comment_item.get("digg_count") if comment_item.get("digg_count") else 0),
"last_modify_ts": utils.get_current_timestamp(),
@@ -229,24 +217,8 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
async def save_creator(user_id: str, creator: Dict):
user_info = creator.get("user", {})
gender_map = {0: "Unknown", 1: "Male", 2: "Female"}
avatar_uri = user_info.get("avatar_300x300", {}).get("uri")
local_db_item = {
"user_id": user_id,
"nickname": user_info.get("nickname"),
"gender": gender_map.get(user_info.get("gender"), "Unknown"),
"avatar": f"https://p3-pc.douyinpic.com/img/{avatar_uri}" + r"~c5_300x300.jpeg?from=2956013662",
"desc": user_info.get("signature"),
"ip_location": user_info.get("ip_location"),
"follows": user_info.get("following_count", 0),
"fans": user_info.get("max_follower_count", 0),
"interaction": user_info.get("total_favorited", 0),
"videos_count": user_info.get("aweme_count", 0),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.douyin.save_creator] creator:{local_db_item}")
await DouyinStoreFactory.create_store().store_creator(local_db_item)
# 教学版：创作者个人资料(昵称/性别/头像/签名/IP/粉丝数等)不再落库，防骚扰。
return
async def update_dy_aweme_image(aweme_id, pic_content, extension_file_name):

39
store/douyin/_store_impl.py
View File

@@ -33,7 +33,7 @@ from sqlalchemy import select
import config
from base.base_crawler import AbstractStore
from database.db_session import get_session
from database.models import DouyinAweme, DouyinAwemeComment, DyCreator
from database.models import DouyinAweme, DouyinAwemeComment
from tools import utils, words
from tools.async_file_writer import AsyncFileWriter
from var import crawler_type_var
@@ -133,24 +133,8 @@ class DouyinDbStoreImplement(AbstractStore):
await session.commit()
async def store_creator(self, creator: Dict):
"""
Douyin creator DB storage implementation
Args:
creator: creator dict
"""
user_id = creator.get("user_id")
async with get_session() as session:
result = await session.execute(select(DyCreator).where(DyCreator.user_id == user_id))
user_detail = result.scalar_one_or_none()
if not user_detail:
creator["add_ts"] = utils.get_current_timestamp()
new_creator = DyCreator(**creator)
session.add(new_creator)
else:
for key, value in creator.items():
setattr(user_detail, key, value)
await session.commit()
# 教学版：创作者个人资料不再落库
pass
class DouyinJsonStoreImplement(AbstractStore):
@@ -275,21 +259,8 @@ class DouyinMongoStoreImplement(AbstractStore):
utils.logger.info(f"[DouyinMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB")
async def store_creator(self, creator_item: Dict):
"""
Store creator information to MongoDB
Args:
creator_item: Creator data
"""
user_id = creator_item.get("user_id")
if not user_id:
return
await self.mongo_store.save_or_update(
collection_suffix="creators",
query={"user_id": user_id},
data=creator_item
)
utils.logger.info(f"[DouyinMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
# 教学版：创作者个人资料不再落库
pass
class DouyinExcelStoreImplement:

34
store/kuaishou/__init__.py
View File

@@ -26,6 +26,7 @@ from typing import List
import config
from var import source_keyword_var
from tools.user_hash import anonymize_user_id, mask_nickname
from ._store_impl import *
@@ -63,9 +64,8 @@ async def update_kuaishou_video(video_item: Dict):
"title": photo_info.get("caption", "")[:500],
"desc": photo_info.get("caption", "")[:500],
"create_time": photo_info.get("timestamp"),
"user_id": user_info.get("id"),
"nickname": user_info.get("name"),
"avatar": user_info.get("headerUrl", ""),
"creator_hash": anonymize_user_id(user_info.get("id")), # 创作者匿名哈希(不存原始 user_id)
"nickname": mask_nickname(user_info.get("name")), # 用户昵称(已脱敏)
"liked_count": str(photo_info.get("realLikeCount")),
"viewd_count": str(photo_info.get("viewCount")),
"last_modify_ts": utils.get_current_timestamp(),
@@ -97,11 +97,10 @@ async def update_ks_video_comment(video_id: str, comment_item: Dict):
"create_time": comment_item.get("timestamp"),
"video_id": video_id,
"content": comment_item.get("content"),
# V2: author_id, Old: authorId
"user_id": comment_item.get("author_id") or comment_item.get("authorId"),
# V2: author_name, Old: authorName
"nickname": comment_item.get("author_name") or comment_item.get("authorName"),
"avatar": comment_item.get("headurl"),
# 创作者匿名哈希(不存原始 user_id)：V2: author_id, Old: authorId
"creator_hash": anonymize_user_id(comment_item.get("author_id") or comment_item.get("authorId")),
# 用户昵称(已脱敏)：V2: author_name, Old: authorName
"nickname": mask_nickname(comment_item.get("author_name") or comment_item.get("authorName")),
# V2: commentCount, Old: subCommentCount
"sub_comment_count": str(comment_item.get("commentCount") or comment_item.get("subCommentCount", 0)),
"last_modify_ts": utils.get_current_timestamp(),
@@ -111,20 +110,5 @@ async def update_ks_video_comment(video_id: str, comment_item: Dict):
await KuaishouStoreFactory.create_store().store_comment(comment_item=save_comment_item)
async def save_creator(user_id: str, creator: Dict):
ownerCount = creator.get('ownerCount', {})
profile = creator.get('profile', {})
local_db_item = {
'user_id': user_id,
'nickname': profile.get('user_name'),
'gender': 'Female' if profile.get('gender') == "F" else 'Male',
'avatar': profile.get('headurl'),
'desc': profile.get('user_text'),
'ip_location': "",
'follows': ownerCount.get("follow"),
'fans': ownerCount.get("fan"),
'interaction': ownerCount.get("photo_public"),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.kuaishou.save_creator] creator:{local_db_item}")
await KuaishouStoreFactory.create_store().store_creator(local_db_item)
# 教学版：创作者个人资料(昵称/性别/头像/签名/IP/粉丝数等)不再落库，防骚扰。
return

17
store/kuaishou/_store_impl.py
View File

@@ -228,21 +228,8 @@ class KuaishouMongoStoreImplement(AbstractStore):
utils.logger.info(f"[KuaishouMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB")
async def store_creator(self, creator_item: Dict):
"""
Store creator information to MongoDB
Args:
creator_item: Creator data
"""
user_id = creator_item.get("user_id")
if not user_id:
return
await self.mongo_store.save_or_update(
collection_suffix="creators",
query={"user_id": user_id},
data=creator_item
)
utils.logger.info(f"[KuaishouMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
# 教学版：创作者个人资料不再落库
pass
class KuaishouExcelStoreImplement:

6
store/tieba/__init__.py
View File

@@ -121,7 +121,5 @@ async def save_creator(user_info: TiebaCreator):
Returns:
"""
local_db_item = user_info.model_dump()
local_db_item["last_modify_ts"] = utils.get_current_timestamp()
utils.logger.info(f"[store.tieba.save_creator] creator:{local_db_item}")
await TieBaStoreFactory.create_store().store_creator(local_db_item)
# 教学版：创作者个人资料不再落库，防骚扰。
return

37
store/tieba/_store_impl.py
View File

@@ -35,7 +35,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
import config
from base.base_crawler import AbstractStore
from database.models import TiebaNote, TiebaComment, TiebaCreator
from database.models import TiebaNote, TiebaComment
from tools import utils, words
from database.db_session import get_session
from var import crawler_type_var
@@ -137,23 +137,8 @@ class TieBaDbStoreImplement(AbstractStore):
await session.commit()
async def store_creator(self, creator: Dict):
"""
tieba content DB storage implementation
Args:
creator: creator dict
"""
user_id = creator.get("user_id")
async with get_session() as session:
stmt = select(TiebaCreator).where(TiebaCreator.user_id == user_id)
res = await session.execute(stmt)
db_creator = res.scalar_one_or_none()
if db_creator:
for key, value in creator.items():
setattr(db_creator, key, value)
else:
db_creator = TiebaCreator(**creator)
session.add(db_creator)
await session.commit()
# 教学版：创作者个人资料不再落库
pass
class TieBaJsonStoreImplement(AbstractStore):
@@ -258,20 +243,8 @@ class TieBaMongoStoreImplement(AbstractStore):
utils.logger.info(f"[TieBaMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB")
async def store_creator(self, creator_item: Dict):
"""
Store creator information to MongoDB
Args:
creator_item: Creator data
"""
user_id = creator_item.get("user_id")
if not user_id:
return
await self.mongo_store.save_or_update(
collection_suffix="creators",
query={"user_id": user_id},
data=creator_item
)
# 教学版：创作者个人资料不再落库
pass
utils.logger.info(f"[TieBaMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")

49
store/weibo/__init__.py
View File

@@ -25,6 +25,7 @@
import re
from typing import List
from tools.user_hash import anonymize_user_id, mask_nickname
from var import source_keyword_var
from .weibo_store_media import *
@@ -78,11 +79,13 @@ async def update_weibo_note(note_item: Dict):
if not note_item:
return
mblog: Dict = note_item.get("mblog")
user_info: Dict = mblog.get("user")
mblog: Dict = note_item.get("mblog") or {}
user_info: Dict = mblog.get("user") or {}
note_id = mblog.get("id")
content_text = mblog.get("text")
clean_text = re.sub(r"<.*?>", "", content_text)
# 教学版：原始 user_id 匿名化为 creator_hash，昵称脱敏；
# 不采集头像/主页链接/性别/IP 归属地等可定位真人的信息。
save_content_item = {
# Weibo information
"note_id": note_id,
@@ -94,14 +97,10 @@ async def update_weibo_note(note_item: Dict):
"shared_count": str(mblog.get("reposts_count", 0)),
"last_modify_ts": utils.get_current_timestamp(),
"note_url": f"https://m.weibo.cn/detail/{note_id}",
"ip_location": mblog.get("region_name", "").replace("发布于 ", ""),
# User information
"user_id": str(user_info.get("id")),
"nickname": user_info.get("screen_name", ""),
"gender": user_info.get("gender", ""),
"profile_url": user_info.get("profile_url", ""),
"avatar": user_info.get("profile_image_url", ""),
# 创作者信息（匿名化/脱敏，不含原始 user_id/avatar/gender/profile_url/ip_location）
"creator_hash": anonymize_user_id(user_info.get("id")),
"nickname": mask_nickname(user_info.get("screen_name", "")),
"source_keyword": source_keyword_var.get(),
}
utils.logger.info(f"[store.weibo.update_weibo_note] weibo note id:{note_id}, title:{save_content_item.get('content')[:24]} ...")
@@ -137,9 +136,11 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict):
if not comment_item or not note_id:
return
comment_id = str(comment_item.get("id"))
user_info: Dict = comment_item.get("user")
user_info: Dict = comment_item.get("user") or {}
content_text = comment_item.get("text")
clean_text = re.sub(r"<.*?>", "", content_text)
# 教学版：原始 user_id 匿名化为 creator_hash，昵称脱敏；
# 不采集头像/主页链接/性别/IP 归属地等可定位真人的信息。
save_comment_item = {
"comment_id": comment_id,
"create_time": utils.rfc2822_to_timestamp(comment_item.get("created_at")),
@@ -149,15 +150,11 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict):
"sub_comment_count": str(comment_item.get("total_number", 0)),
"comment_like_count": str(comment_item.get("like_count", 0)),
"last_modify_ts": utils.get_current_timestamp(),
"ip_location": comment_item.get("source", "").replace("来自", ""),
"parent_comment_id": comment_item.get("rootid", ""),
# User information
"user_id": str(user_info.get("id")),
"nickname": user_info.get("screen_name", ""),
"gender": user_info.get("gender", ""),
"profile_url": user_info.get("profile_url", ""),
"avatar": user_info.get("profile_image_url", ""),
# 创作者信息（匿名化/脱敏，不含原始 user_id/avatar/gender/profile_url/ip_location）
"creator_hash": anonymize_user_id(user_info.get("id")),
"nickname": mask_nickname(user_info.get("screen_name", "")),
}
utils.logger.info(f"[store.weibo.update_weibo_note_comment] Weibo note comment: {comment_id}, content: {save_comment_item.get('content', '')[:24]} ...")
await WeibostoreFactory.create_store().store_comment(comment_item=save_comment_item)
@@ -180,6 +177,8 @@ async def update_weibo_note_image(picid: str, pic_content, extension_file_name):
async def save_creator(user_id: str, user_info: Dict):
"""
Save creator information to local
教学版：为防骚扰不再采集/持久化创作者个人信息（昵称/性别/头像/简介/IP/粉丝数等），
此入口保留为空操作以兼容调用方。user_id 仅在调用方局部用于抓取该创作者的微博。
Args:
user_id:
user_info:
@@ -187,17 +186,5 @@ async def save_creator(user_id: str, user_info: Dict):
Returns:
"""
local_db_item = {
'user_id': user_id,
'nickname': user_info.get('screen_name'),
'gender': 'Female' if user_info.get('gender') == "f" else 'Male',
'avatar': user_info.get('avatar_hd'),
'desc': user_info.get('description'),
'ip_location': user_info.get("source", "").replace("来自", ""),
'follows': user_info.get('follow_count', ''),
'fans': user_info.get('followers_count', ''),
'tag_list': '',
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.weibo.save_creator] creator:{local_db_item}")
await WeibostoreFactory.create_store().store_creator(local_db_item)
# 教学版：创作者个人信息均不采集不持久化
return

53
store/weibo/_store_impl.py
View File

@@ -35,7 +35,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
import config
from base.base_crawler import AbstractStore
from database.models import WeiboCreator, WeiboNote, WeiboNoteComment
from database.models import WeiboNote, WeiboNoteComment
from tools import utils, words
from tools.async_file_writer import AsyncFileWriter
from database.db_session import get_session
@@ -58,6 +58,13 @@ def calculate_number_of_files(file_store_path: str) -> int:
return 1
def _filter_model_fields(model_cls, item: Dict) -> Dict:
"""只保留目标 ORM 模型已有的列，避免把已删除/多余字段（如 avatar/gender/
profile_url/ip_location/user_id）传给 ORM 构造而报错。教学版兜底保护。"""
allowed = {col.name for col in model_cls.__table__.columns}
return {k: v for k, v in item.items() if k in allowed}
class WeiboCsvStoreImplement(AbstractStore):
def __init__(self, **kwargs):
super().__init__(**kwargs)
@@ -88,13 +95,14 @@ class WeiboCsvStoreImplement(AbstractStore):
async def store_creator(self, creator: Dict):
"""
Weibo creator CSV storage implementation
教学版：不采集/持久化创作者个人信息，空操作。
Args:
creator:
Returns:
"""
await self.writer.write_to_csv(item_type="creators", item=creator)
pass
class WeiboDbStoreImplement(AbstractStore):
@@ -108,6 +116,8 @@ class WeiboDbStoreImplement(AbstractStore):
Returns:
"""
# 教学版兜底：过滤掉已删除/多余字段，确保不会把 user_id/avatar 等传给 ORM
content_item = _filter_model_fields(WeiboNote, content_item)
note_id = int(content_item.get("note_id"))
content_item["note_id"] = note_id
async with get_session() as session:
@@ -135,6 +145,8 @@ class WeiboDbStoreImplement(AbstractStore):
Returns:
"""
# 教学版兜底：过滤掉已删除/多余字段，确保不会把 user_id/avatar 等传给 ORM
comment_item = _filter_model_fields(WeiboNoteComment, comment_item)
comment_id = int(comment_item.get("comment_id"))
comment_item["comment_id"] = comment_id
comment_item["note_id"] = int(comment_item.get("note_id", 0) or 0)
@@ -162,29 +174,14 @@ class WeiboDbStoreImplement(AbstractStore):
async def store_creator(self, creator: Dict):
"""
Weibo creator DB storage implementation
教学版：不采集/持久化创作者个人信息，空操作（WeiboCreator 表已删除）。
Args:
creator:
Returns:
"""
user_id = int(creator.get("user_id"))
creator["user_id"] = user_id
async with get_session() as session:
stmt = select(WeiboCreator).where(WeiboCreator.user_id == user_id)
res = await session.execute(stmt)
db_creator = res.scalar_one_or_none()
if db_creator:
db_creator.last_modify_ts = utils.get_current_timestamp()
for key, value in creator.items():
if hasattr(db_creator, key):
setattr(db_creator, key, value)
else:
creator["add_ts"] = utils.get_current_timestamp()
creator["last_modify_ts"] = utils.get_current_timestamp()
db_creator = WeiboCreator(**creator)
session.add(db_creator)
await session.commit()
pass
class WeiboJsonStoreImplement(AbstractStore):
@@ -217,13 +214,14 @@ class WeiboJsonStoreImplement(AbstractStore):
async def store_creator(self, creator: Dict):
"""
creator JSON storage implementation
教学版：不采集/持久化创作者个人信息，空操作。
Args:
creator:
Returns:
"""
await self.writer.write_single_item_to_json(item_type="creators", item=creator)
pass
class WeiboJsonlStoreImplement(AbstractStore):
@@ -238,7 +236,8 @@ class WeiboJsonlStoreImplement(AbstractStore):
await self.writer.write_to_jsonl(item_type="comments", item=comment_item)
async def store_creator(self, creator: Dict):
await self.writer.write_to_jsonl(item_type="creators", item=creator)
# 教学版：不采集/持久化创作者个人信息，空操作。
pass
class WeiboSqliteStoreImplement(WeiboDbStoreImplement):
@@ -291,19 +290,11 @@ class WeiboMongoStoreImplement(AbstractStore):
async def store_creator(self, creator_item: Dict):
"""
Store creator information to MongoDB
教学版：不采集/持久化创作者个人信息，空操作。
Args:
creator_item: Creator data
"""
user_id = creator_item.get("user_id")
if not user_id:
return
await self.mongo_store.save_or_update(
collection_suffix="creators",
query={"user_id": user_id},
data=creator_item
)
utils.logger.info(f"[WeiboMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
pass
class WeiboExcelStoreImplement:

52
store/xhs/__init__.py
View File

@@ -25,6 +25,7 @@ from typing import List
import config
from var import source_keyword_var
from tools.user_hash import anonymize_user_id, mask_nickname
from .xhs_store_media import *
from ._store_impl import *
@@ -113,14 +114,12 @@ async def update_xhs_note(note_item: Dict):
"video_url": video_url, # Note video url
"time": note_item.get("time"), # Note publish time
"last_update_time": note_item.get("last_update_time", 0), # Note last update time
"user_id": user_info.get("user_id"), # User ID
"nickname": user_info.get("nickname"), # User nickname
"avatar": user_info.get("avatar"), # User avatar
"creator_hash": anonymize_user_id(user_info.get("user_id")), # 创作者匿名哈希(不存原始 user_id)
"nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏)
"liked_count": interact_info.get("liked_count"), # Like count
"collected_count": interact_info.get("collected_count"), # Collection count
"comment_count": interact_info.get("comment_count"), # Comment count
"share_count": interact_info.get("share_count"), # Share count
"ip_location": note_item.get("ip_location", ""), # IP location
"image_list": ','.join([img.get('url', '') for img in image_list]), # Image URLs
"tag_list": ','.join([tag.get('name', '') for tag in tag_list if tag.get('type') == 'topic']), # Tags
"last_modify_ts": utils.get_current_timestamp(), # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage)
@@ -165,12 +164,10 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
local_db_item = {
"comment_id": comment_id, # Comment ID
"create_time": comment_item.get("create_time"), # Comment time
"ip_location": comment_item.get("ip_location"), # IP location
"note_id": note_id, # Note ID
"content": comment_item.get("content"), # Comment content
"user_id": user_info.get("user_id"), # User ID
"nickname": user_info.get("nickname"), # User nickname
"avatar": user_info.get("image"), # User avatar
"creator_hash": anonymize_user_id(user_info.get("user_id")), # 创作者匿名哈希(不存原始 user_id)
"nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏)
"sub_comment_count": comment_item.get("sub_comment_count", 0), # Sub-comment count
"pictures": ",".join(comment_pictures), # Comment pictures
"parent_comment_id": target_comment.get("id", 0), # Parent comment ID
@@ -191,43 +188,8 @@ async def save_creator(user_id: str, creator: Dict):
Returns:
"""
user_info = creator.get('basicInfo', {})
follows = 0
fans = 0
interaction = 0
for i in creator.get('interactions'):
if i.get('type') == 'follows':
follows = i.get('count')
elif i.get('type') == 'fans':
fans = i.get('count')
elif i.get('type') == 'interaction':
interaction = i.get('count')
def get_gender(gender):
if gender == 1:
return 'Female'
elif gender == 0:
return 'Male'
else:
return None
local_db_item = {
'user_id': user_id, # User ID
'nickname': user_info.get('nickname'), # Nickname
'gender': get_gender(user_info.get('gender')), # Gender
'avatar': user_info.get('images'), # Avatar
'desc': user_info.get('desc'), # Personal description
'ip_location': user_info.get('ipLocation'), # IP location
'follows': follows, # Following count
'fans': fans, # Fans count
'interaction': interaction, # Interaction count
'tag_list': json.dumps({tag.get('tagType'): tag.get('name')
for tag in creator.get('tags')}, ensure_ascii=False), # Tags
"last_modify_ts": utils.get_current_timestamp(), # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage)
}
utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}")
await XhsStoreFactory.create_store().store_creator(local_db_item)
# 教学版：创作者个人资料(昵称/性别/头像/IP/粉丝数等)不再落库，防骚扰。
return
async def update_xhs_note_image(note_id, pic_content, extension_file_name):

72
store/xhs/_store_impl.py
View File

@@ -30,7 +30,7 @@ from sqlalchemy.orm import Session
from base.base_crawler import AbstractStore
from database.db_session import get_session
from database.models import XhsNote, XhsNoteComment, XhsCreator
from database.models import XhsNote, XhsNoteComment
from tools.async_file_writer import AsyncFileWriter
from tools.time_util import get_current_timestamp
@@ -137,10 +137,8 @@ class XhsDbStoreImplement(AbstractStore):
add_ts = int(get_current_timestamp())
last_modify_ts = int(get_current_timestamp())
note = XhsNote(
user_id=content_item.get("user_id"),
creator_hash=content_item.get("creator_hash"),
nickname=content_item.get("nickname"),
avatar=content_item.get("avatar"),
ip_location=content_item.get("ip_location"),
add_ts=add_ts,
last_modify_ts=last_modify_ts,
note_id=content_item.get("note_id"),
@@ -197,10 +195,8 @@ class XhsDbStoreImplement(AbstractStore):
add_ts = int(get_current_timestamp())
last_modify_ts = int(get_current_timestamp())
comment = XhsNoteComment(
user_id=comment_item.get("user_id"),
creator_hash=comment_item.get("creator_hash"),
nickname=comment_item.get("nickname"),
avatar=comment_item.get("avatar"),
ip_location=comment_item.get("ip_location"),
add_ts=add_ts,
last_modify_ts=last_modify_ts,
comment_id=comment_item.get("comment_id"),
@@ -231,54 +227,8 @@ class XhsDbStoreImplement(AbstractStore):
return result.first() is not None
async def store_creator(self, creator_item: Dict):
user_id = creator_item.get("user_id")
if not user_id:
return
async with get_session() as session:
if await self.creator_is_exist(session, user_id):
await self.update_creator(session, creator_item)
else:
await self.add_creator(session, creator_item)
async def add_creator(self, session: AsyncSession, creator_item: Dict):
add_ts = int(get_current_timestamp())
last_modify_ts = int(get_current_timestamp())
creator = XhsCreator(
user_id=creator_item.get("user_id"),
nickname=creator_item.get("nickname"),
avatar=creator_item.get("avatar"),
ip_location=creator_item.get("ip_location"),
add_ts=add_ts,
last_modify_ts=last_modify_ts,
desc=creator_item.get("desc"),
gender=creator_item.get("gender"),
follows=str(creator_item.get("follows")),
fans=str(creator_item.get("fans")),
interaction=str(creator_item.get("interaction")),
tag_list=json.dumps(creator_item.get("tag_list"))
)
session.add(creator)
async def update_creator(self, session: AsyncSession, creator_item: Dict):
user_id = creator_item.get("user_id")
last_modify_ts = int(get_current_timestamp())
update_data = {
"last_modify_ts": last_modify_ts,
"nickname": creator_item.get("nickname"),
"avatar": creator_item.get("avatar"),
"desc": creator_item.get("desc"),
"follows": str(creator_item.get("follows")),
"fans": str(creator_item.get("fans")),
"interaction": str(creator_item.get("interaction")),
"tag_list": json.dumps(creator_item.get("tag_list"))
}
stmt = update(XhsCreator).where(XhsCreator.user_id == user_id).values(**update_data)
await session.execute(stmt)
async def creator_is_exist(self, session: AsyncSession, user_id: str) -> bool:
stmt = select(XhsCreator).where(XhsCreator.user_id == user_id)
result = await session.execute(stmt)
return result.first() is not None
# 教学版：创作者个人资料不再落库
pass
async def get_all_content(self) -> List[Dict]:
async with get_session() as session:
@@ -345,16 +295,8 @@ class XhsMongoStoreImplement(AbstractStore):
Args:
creator_item: Creator data
"""
user_id = creator_item.get("user_id")
if not user_id:
return
await self.mongo_store.save_or_update(
collection_suffix="creators",
query={"user_id": user_id},
data=creator_item
)
utils.logger.info(f"[XhsMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
# 教学版：创作者个人资料不再落库
pass
class XhsExcelStoreImplement:

66
store/zhihu/_store_impl.py
View File

@@ -36,7 +36,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
import config
from base.base_crawler import AbstractStore
from database.db_session import get_session
from database.models import ZhihuContent, ZhihuComment, ZhihuCreator
from database.models import ZhihuContent, ZhihuComment
from tools import utils, words
from var import crawler_type_var
from tools.async_file_writer import AsyncFileWriter
@@ -85,15 +85,8 @@ class ZhihuCsvStoreImplement(AbstractStore):
await self.writer.write_to_csv(item_type="comments", item=comment_item)
async def store_creator(self, creator: Dict):
"""
Zhihu content CSV storage implementation
Args:
creator: creator dict
Returns:
"""
await self.writer.write_to_csv(item_type="creators", item=creator)
"""Creator profile is no longer persisted (teaching version: anti-harassment)."""
pass
class ZhihuDbStoreImplement(AbstractStore):
@@ -142,26 +135,8 @@ class ZhihuDbStoreImplement(AbstractStore):
await session.commit()
async def store_creator(self, creator: Dict):
"""
Zhihu content DB storage implementation
Args:
creator: creator dict
"""
user_id = creator.get("user_id")
async with get_session() as session:
stmt = select(ZhihuCreator).where(ZhihuCreator.user_id == user_id)
result = await session.execute(stmt)
existing_creator = result.scalars().first()
if existing_creator:
for key, value in creator.items():
if hasattr(existing_creator, key):
setattr(existing_creator, key, value)
else:
if "add_ts" not in creator:
creator["add_ts"] = utils.get_current_timestamp()
new_creator = ZhihuCreator(**creator)
session.add(new_creator)
await session.commit()
"""Creator profile is no longer persisted (teaching version: anti-harassment)."""
pass
class ZhihuJsonStoreImplement(AbstractStore):
@@ -192,15 +167,8 @@ class ZhihuJsonStoreImplement(AbstractStore):
await self.writer.write_single_item_to_json(item_type="comments", item=comment_item)
async def store_creator(self, creator: Dict):
"""
Zhihu content JSON storage implementation
Args:
creator: creator dict
Returns:
"""
await self.writer.write_single_item_to_json(item_type="creators", item=creator)
"""Creator profile is no longer persisted (teaching version: anti-harassment)."""
pass
class ZhihuJsonlStoreImplement(AbstractStore):
@@ -215,7 +183,8 @@ class ZhihuJsonlStoreImplement(AbstractStore):
await self.writer.write_to_jsonl(item_type="comments", item=comment_item)
async def store_creator(self, creator: Dict):
await self.writer.write_to_jsonl(item_type="creators", item=creator)
"""Creator profile is no longer persisted (teaching version: anti-harassment)."""
pass
class ZhihuSqliteStoreImplement(ZhihuDbStoreImplement):
@@ -266,21 +235,8 @@ class ZhihuMongoStoreImplement(AbstractStore):
utils.logger.info(f"[ZhihuMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB")
async def store_creator(self, creator_item: Dict):
"""
Store creator information to MongoDB
Args:
creator_item: Creator data
"""
user_id = creator_item.get("user_id")
if not user_id:
return
await self.mongo_store.save_or_update(
collection_suffix="creators",
query={"user_id": user_id},
data=creator_item
)
utils.logger.info(f"[ZhihuMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
"""Creator profile is no longer persisted (teaching version: anti-harassment)."""
pass
class ZhihuExcelStoreImplement:

362
tests/test_douyin_no_user_info.py Normal file
View File

@@ -0,0 +1,362 @@
# -*- coding: utf-8 -*-
"""
教学版回归测试(抖音 douyin):确保 store/douyin 存储链路不再持久化可定位真人的
用户个人信息。
教学版约定:用户 ID(uid/sec_uid/short_user_id/user_unique_id)/IP/头像/签名/性别
一律不持久化;原始 uid 经 tools.user_hash.anonymize_user_id 转成 creator_hash
(sha256 截断 16 位)写入;昵称保留但经 mask_nickname 中间脱敏。desc(作品描述)是
内容字段,保留。
覆盖:
1. test_douyin_aweme_masks_user_info
- mock aweme_item(含 author.uid/sec_uid/short_id/unique_id/nickname/
avatar_thumb.url_list/signature、aweme_item.ip_label、statistics、video/
music/images 等内容字段)
- FakeStore 捕获 update_douyin_aweme 产出的存储 dict
- 断言:不含禁用键;含 creator_hash(≠原 uid);nickname 脱敏≠原文;
原始敏感值不在任何存储值中;内容字段正常提取
2. test_douyin_comment_masks_user_info
- 同上,针对 update_dy_aweme_comment(含嵌套 user.* / comment_item.ip_label /
cid / text / image_list 等)
3. test_douyin_store_end_to_end_sqlite
- FakeStore 捕获真实 dict → DouyinAweme(**captured) 构造(ORM 列校验:
dict 多了已删列会 TypeError)
- 端到端:monkeypatch db_session 为内存 SQLite engine + create_all,
走真实 DouyinDbStoreImplement.store_content(含 int(aweme_id) / if title
判断 / DouyinAweme(**content_item)),查询回读,验证 dict key 与删列后 ORM
完全对得上
"""
import asyncio
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import StaticPool
import config
import store.douyin as ds
from database import db_session
from database.models import Base, DouyinAweme, DouyinAwemeComment
from tools.user_hash import anonymize_user_id, mask_nickname
# 抖音教学版禁用字段(键):不得作为存储 dict 的 key 出现。
FORBIDDEN_KEYS = {
"user_id", "sec_uid", "short_user_id", "user_unique_id",
"avatar", "user_signature", "ip_location",
}
# ----------------------------- mock payload -----------------------------
def _build_aweme_item() -> dict:
"""贴近真实抖音结构的 mock aweme_item。
含会被拍平的用户字段(uid/sec_uid/short_id/unique_id/avatar/signature)与
ip_label,这些必须不落库;同时含 statistics/video/music/images 等内容字段,
让 _extract_* 辅助函数返回非空值。"""
return {
"aweme_id": "7234567890123456",
"aweme_type": 0,
"desc": "这是一个作品描述,同时作为 title",
"create_time": 1700000000,
"ip_label": "上海", # IP 归属地,禁用
"author": {
"uid": "9876543210",
"sec_uid": "MS4wLjABAAAASecretSecUidForTest",
"short_id": "88877766",
"unique_id": "creator_unique_abc",
"nickname": "抖音创作者",
"avatar_thumb": {"url_list": ["http://x/avatar_thumb.jpg"]},
"avatar_medium": {"url_list": ["http://x/avatar_medium.jpg"]},
"signature": "这是创作者个人签名内容",
},
"statistics": {
"digg_count": 100,
"collect_count": 5,
"comment_count": 20,
"share_count": 3,
},
"video": {
"raw_cover": {"url_list": ["", "http://x/cover.jpg"]},
"origin_cover": {"url_list": ["", "http://x/cover2.jpg"]},
"play_addr_h264": {"url_list": ["", "http://x/video_h264.mp4"]},
"play_addr_256": {"url_list": ["", "http://x/video_256.mp4"]},
"play_addr": {"url_list": ["", "http://x/video.mp4"]},
},
"music": {
"play_url": {"uri": "http://x/music.mp3"},
},
"images": [
{"url_list": ["", "http://x/note_img1.jpg"]},
{"url_list": ["", "http://x/note_img2.jpg"]},
],
}
def _build_comment_item() -> dict:
"""贴近真实抖音结构的 mock comment_item。aweme_id 须与传入
update_dy_aweme_comment 的 aweme_id 一致。"""
return {
"aweme_id": "7234567890123456",
"cid": "1111111111",
"text": "这是一条评论内容",
"create_time": 1700000001,
"reply_id": "0",
"reply_comment_total": 2,
"digg_count": 5,
"ip_label": "北京", # IP 归属地,禁用
"user": {
"uid": "555666777",
"sec_uid": "MS4wLjABBBBCommentSecUid",
"short_id": "22233344",
"unique_id": "commenter_unique_xyz",
"nickname": "评论员小王",
"avatar_medium": {"url_list": ["http://x/cavatar.jpg"]},
"signature": "评论员个人签名内容",
},
"image_list": [
{"origin_url": {"url_list": ["", "http://x/cimg.jpg"]}},
],
}
# ----------------------------- 辅助 -----------------------------
class _FakeStore:
"""捕获存储 dict,不真正落库。"""
def __init__(self):
self.contents = []
self.comments = []
async def store_content(self, content_item):
self.contents.append(dict(content_item))
async def store_comment(self, comment_item):
self.comments.append(dict(comment_item))
def _patch_factory(fake):
"""把 DouyinStoreFactory.create_store 替换为返回 fake,返回原方法用于还原。"""
orig = ds.DouyinStoreFactory.create_store
ds.DouyinStoreFactory.create_store = staticmethod(lambda: fake)
return orig
def _assert_no_forbidden(captured: dict, label: str):
hit = set(captured.keys()) & FORBIDDEN_KEYS
assert not hit, f"[{label}] 存储 dict 仍含禁用键: {hit}"
def _assert_raw_values_absent(captured: dict, raw_values, label: str):
"""禁用的原始敏感值(uid/sec_uid/头像/签名/IP 等)不得出现在任何存储值里。
creator_hash 是由 uid 派生的匿名哈希(已单独断言 ≠ 原文),不参与子串扫描。"""
leaked = []
for rv in raw_values:
if not rv:
continue
for k, v in captured.items():
if k == "creator_hash":
continue
if isinstance(v, str) and rv in v:
leaked.append((k, rv))
assert not leaked, f"[{label}] 存储 dict 中泄漏了原始敏感值: {leaked}"
# ----------------------------- 测试 -----------------------------
def test_douyin_aweme_masks_user_info():
aweme = _build_aweme_item()
raw_uid = aweme["author"]["uid"]
raw_nick = aweme["author"]["nickname"]
raw_sensitive = [
raw_uid,
aweme["author"]["sec_uid"],
aweme["author"]["short_id"],
aweme["author"]["unique_id"],
aweme["author"]["avatar_thumb"]["url_list"][0],
aweme["author"]["signature"],
aweme["ip_label"],
]
fake = _FakeStore()
orig = _patch_factory(fake)
try:
asyncio.run(ds.update_douyin_aweme(aweme))
finally:
ds.DouyinStoreFactory.create_store = orig
assert len(fake.contents) == 1
captured = fake.contents[0]
# 1. 禁用键不出现
_assert_no_forbidden(captured, "douyin_aweme")
# 2. 原始敏感值不泄漏到任何存储值
_assert_raw_values_absent(captured, raw_sensitive, "douyin_aweme")
# 3. creator_hash 存在、≠原 uid、与 anonymize_user_id 一致
assert captured.get("creator_hash")
assert captured["creator_hash"] != raw_uid
assert captured["creator_hash"] == anonymize_user_id(raw_uid)
# 4. nickname 保留但脱敏,≠原文,与 mask_nickname 一致
assert captured.get("nickname")
assert captured["nickname"] != raw_nick
assert captured["nickname"] == mask_nickname(raw_nick)
# 5. 内容字段保留(desc/title 是作品描述,不禁用)
assert captured.get("desc") == aweme["desc"]
assert captured.get("title") == aweme["desc"]
# 6. _extract_* 内容字段正常提取(非空)
assert captured.get("cover_url") == "http://x/cover.jpg"
assert captured.get("video_download_url") == "http://x/video_h264.mp4"
assert captured.get("music_download_url") == "http://x/music.mp3"
assert "http://x/note_img1.jpg" in captured.get("note_download_url", "")
# 7. 互动数据拍平
assert captured.get("liked_count") == "100"
assert captured.get("collected_count") == "5"
assert captured.get("comment_count") == "20"
assert captured.get("share_count") == "3"
assert captured.get("aweme_url") == f"https://www.douyin.com/video/{aweme['aweme_id']}"
def test_douyin_comment_masks_user_info():
aweme_id = "7234567890123456"
comment = _build_comment_item()
raw_uid = comment["user"]["uid"]
raw_nick = comment["user"]["nickname"]
raw_sensitive = [
raw_uid,
comment["user"]["sec_uid"],
comment["user"]["short_id"],
comment["user"]["unique_id"],
comment["user"]["avatar_medium"]["url_list"][0],
comment["user"]["signature"],
comment["ip_label"],
]
fake = _FakeStore()
orig = _patch_factory(fake)
try:
asyncio.run(ds.update_dy_aweme_comment(aweme_id, comment))
finally:
ds.DouyinStoreFactory.create_store = orig
assert len(fake.comments) == 1
captured = fake.comments[0]
_assert_no_forbidden(captured, "douyin_comment")
_assert_raw_values_absent(captured, raw_sensitive, "douyin_comment")
assert captured.get("creator_hash")
assert captured["creator_hash"] != raw_uid
assert captured["creator_hash"] == anonymize_user_id(raw_uid)
assert captured.get("nickname")
assert captured["nickname"] != raw_nick
assert captured["nickname"] == mask_nickname(raw_nick)
# 评论内容/ID 保留
assert captured.get("content") == comment["text"]
assert captured.get("comment_id") == comment["cid"]
assert captured.get("aweme_id") == aweme_id
assert captured.get("parent_comment_id") == "0"
assert captured.get("sub_comment_count") == "2"
# 评论图片提取
assert captured.get("pictures") == "http://x/cimg.jpg"
def test_douyin_store_end_to_end_sqlite(monkeypatch):
aweme = _build_aweme_item()
raw_uid = aweme["author"]["uid"]
raw_nick = aweme["author"]["nickname"]
# ---- 1. FakeStore 捕获 update_douyin_aweme 产出的真实 dict ----
fake = _FakeStore()
orig = _patch_factory(fake)
try:
asyncio.run(ds.update_douyin_aweme(aweme))
finally:
ds.DouyinStoreFactory.create_store = orig
assert len(fake.contents) == 1
captured = fake.contents[0]
# ---- 2. DouyinAweme(**captured) 构造校验 ----
# dict 多了已删列会 TypeError,少了非空必填列 SQLAlchemy 也会报错;
# 此处证明 captured 的 key 与删列后 ORM 列完全对得上,不抛异常。
obj = DouyinAweme(**captured)
assert int(obj.aweme_id) == int(aweme["aweme_id"])
assert obj.creator_hash == anonymize_user_id(raw_uid)
assert obj.creator_hash != raw_uid
assert obj.nickname == mask_nickname(raw_nick)
assert obj.nickname != raw_nick
assert obj.title == aweme["desc"]
assert obj.desc == aweme["desc"]
_assert_no_forbidden(captured, "douyin_aweme_orm_construct")
# ---- 3. 端到端:内存 SQLite + 真实 store_content ----
# 用 StaticPool 保证 :memory: 库在同一个连接上持久(跨 session 可见)。
engine = create_async_engine(
"sqlite+aiosqlite:///:memory:",
poolclass=StaticPool,
)
# 让 db_session 用内存 engine;SAVE_DATA_OPTION=db 让工厂走 DouyinDbStoreImplement
monkeypatch.setattr(db_session, "get_async_engine", lambda *a, **kw: engine)
monkeypatch.setattr(config, "SAVE_DATA_OPTION", "db")
async def _scenario():
# 建表
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
# 真实 store_content 路径(含 int(aweme_id) 与 if content_item.get("title") 判断)
await ds.update_douyin_aweme(aweme)
# 查询回读
async with db_session.get_session() as session:
res = await session.execute(
select(DouyinAweme).where(DouyinAweme.aweme_id == int(aweme["aweme_id"]))
)
row = res.scalar_one_or_none()
await engine.dispose()
return row
row = asyncio.run(_scenario())
# ---- 4. 回读断言 ----
assert row is not None, "作品未写入 SQLite"
assert int(row.aweme_id) == int(aweme["aweme_id"])
assert row.creator_hash == anonymize_user_id(raw_uid)
assert row.creator_hash != raw_uid
assert row.nickname == mask_nickname(raw_nick)
assert row.nickname != raw_nick
assert row.desc == aweme["desc"]
assert row.title == aweme["desc"]
assert row.cover_url == "http://x/cover.jpg"
assert row.video_download_url == "http://x/video_h264.mp4"
assert row.music_download_url == "http://x/music.mp3"
# 禁用列在 ORM 上不存在(自省)
orm_cols = {c.name for c in DouyinAweme.__table__.columns}
assert not (orm_cols & FORBIDDEN_KEYS), f"ORM 仍含禁用列: {orm_cols & FORBIDDEN_KEYS}"
# captured 的所有 key 都是合法 ORM 列(无悬空 key)
assert set(captured.keys()).issubset(orm_cols), (
f"captured 含非 ORM 列: {set(captured.keys()) - orm_cols}"
)
# 评论同样做一次 ORM 构造校验(证明 comment dict key 对得上)
comment = _build_comment_item()
fake_c = _FakeStore()
orig_c = _patch_factory(fake_c)
try:
asyncio.run(ds.update_dy_aweme_comment(comment["aweme_id"], comment))
finally:
ds.DouyinStoreFactory.create_store = orig_c
captured_comment = fake_c.comments[0]
comment_obj = DouyinAwemeComment(**captured_comment) # 不抛异常即对得上
assert int(comment_obj.comment_id) == int(comment["cid"])
assert comment_obj.creator_hash == anonymize_user_id(comment["user"]["uid"])
assert comment_obj.nickname == mask_nickname(comment["user"]["nickname"])
_assert_no_forbidden(captured_comment, "douyin_comment_orm_construct")
comment_orm_cols = {c.name for c in DouyinAwemeComment.__table__.columns}
assert set(captured_comment.keys()).issubset(comment_orm_cols), (
f"captured_comment 含非 ORM 列: {set(captured_comment.keys()) - comment_orm_cols}"
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])

284
tests/test_kuaishou_no_user_info.py Normal file
View File

@@ -0,0 +1,284 @@
# -*- coding: utf-8 -*-
"""
教学版回归测试：快手(kuaishou)存储链路不再持久化可定位真人的用户个人信息。
覆盖：
1. update_kuaishou_video —— mock video_item(含 author.id/name/headerUrl + photo 内容字段 + type)
经 FakeStore 捕获，断言捕获 dict 不含禁用键(user_id/avatar/signature/ip_location/gender)、
含 creator_hash(≠原 user_id)、nickname 脱敏(≠原文)。
2. update_ks_video_comment —— V2(snake_case) 与旧 GraphQL(camelCase) 两种 comment_item 格式各测一次。
3. test_kuaishou_store_end_to_end_sqlite —— 端到端：FakeStore 捕获真实 dict ->
KuaishouVideo(**captured) 触发 ORM 列校验 -> 写入内存 SQLite -> 查询回读校验属性。
约束：只新建本测试文件，只读源码不改源码；不依赖网络/登录。
"""
import asyncio
import contextlib
import types
import pytest
import store.kuaishou as ks
from store.kuaishou import update_kuaishou_video, update_ks_video_comment
from tools.user_hash import anonymize_user_id, mask_nickname
# 教学版禁用字段(键)：一律不得出现在存储 dict 中。
# 昵称字段 nickname 允许保留，但值须脱敏。
FORBIDDEN_KEYS = {"user_id", "avatar", "signature", "ip_location", "gender"}
# ----------------------------- mock 数据 -----------------------------
MOCK_VIDEO_ID = "3xf8e9kq2b7w4"
MOCK_AUTHOR_ID = "ks_author_001"
MOCK_AUTHOR_NAME = "快手达人"
MOCK_CAPTION = "这是一条测试视频，教学版脱敏回归 #测试"
# 评论作者(两种格式用不同 id/昵称，便于分别断言)
MOCK_COMMENT_V2_AUTHOR_ID = "ks_user_888"
MOCK_COMMENT_V2_AUTHOR_NAME = "快手老铁"
MOCK_COMMENT_LEGACY_AUTHOR_ID = "ks_user_777"
MOCK_COMMENT_LEGACY_AUTHOR_NAME = "快乐源泉"
def make_mock_video() -> dict:
"""贴近真实快手结构的 video_item：author 在顶层，photo 含内容字段。
author.headerUrl 与各禁用字段一样不应进入存储 dict。"""
return {
"type": 1,
"photo": {
"id": MOCK_VIDEO_ID,
"caption": MOCK_CAPTION,
"timestamp": 1700000000000,
"coverUrl": "https://p.kuaishou.com/cover/abc.jpg",
"photoUrl": "https://v.kuaishou.com/play/abc.mp4",
"realLikeCount": 12345,
"viewCount": 67890,
},
"author": {
"id": MOCK_AUTHOR_ID,
"name": MOCK_AUTHOR_NAME,
"headerUrl": "https://p.kuaishou.com/header/u001.jpg",
},
}
def make_mock_comment_v2() -> dict:
"""V2 API 格式：snake_case 字段名，comment_id 为 int。"""
return {
"comment_id": 9001,
"timestamp": 1700000001234,
"content": "太搞笑了哈哈哈",
"author_id": MOCK_COMMENT_V2_AUTHOR_ID,
"author_name": MOCK_COMMENT_V2_AUTHOR_NAME,
"headurl": "https://p.kuaishou.com/header/u888.jpg",
"commentCount": 7,
}
def make_mock_comment_legacy() -> dict:
"""旧 GraphQL API 格式：camelCase 字段名。"""
return {
"commentId": 8001,
"timestamp": 1700000005678,
"content": "这条评论来自旧 GraphQL 接口",
"authorId": MOCK_COMMENT_LEGACY_AUTHOR_ID,
"authorName": MOCK_COMMENT_LEGACY_AUTHOR_NAME,
"headurl": "https://p.kuaishou.com/header/u777.jpg",
"subCommentCount": 3,
}
# ----------------------------- FakeStore 捕获 -----------------------------
@contextlib.contextmanager
def _patch_create_store():
"""把 KuaishouStoreFactory.create_store 替换为返回捕获用 FakeStore 的 staticmethod。
FakeStore 把 store_content/store_comment 收到的 dict 原样写入 holder.content / holder.comment。
保存/还原走类 __dict__ 中的 staticmethod 描述符本身，避免把 staticmethod 退化为普通方法
而污染后续测试。"""
holder = types.SimpleNamespace(content={}, comment={})
class FakeStore:
async def store_content(self, content_item):
holder.content.clear()
holder.content.update(content_item)
async def store_comment(self, comment_item):
holder.comment.clear()
holder.comment.update(comment_item)
async def store_creator(self, creator):
pass
orig = ks.KuaishouStoreFactory.__dict__["create_store"]
ks.KuaishouStoreFactory.create_store = staticmethod(lambda: FakeStore())
try:
yield holder
finally:
ks.KuaishouStoreFactory.create_store = orig
def _assert_no_forbidden_keys(d: dict, label: str):
hit = set(d.keys()) & FORBIDDEN_KEYS
assert not hit, f"[{label}] 输出仍含禁用字段键: {hit}"
# ----------------------------- 测试 -----------------------------
def test_kuaishou_video_masks_user_info():
"""video 链路：原始 user_id 转 creator_hash、昵称脱敏、headerUrl/禁用键不落库。"""
with _patch_create_store() as holder:
asyncio.run(update_kuaishou_video(make_mock_video()))
captured = holder.content
assert captured, "FakeStore 未捕获到 content_item"
_assert_no_forbidden_keys(captured, "kuaishou_video")
# 头像字段不应进入存储 dict(author.headerUrl 已被丢弃)
assert "headerUrl" not in captured and "avatar" not in captured
# creator_hash 存在且不等于原始 user_id
assert captured.get("creator_hash") == anonymize_user_id(MOCK_AUTHOR_ID)
assert captured["creator_hash"] != MOCK_AUTHOR_ID
assert captured["creator_hash"] # 非空
# 昵称已脱敏：等于 mask_nickname(原文) 且不等于原文
assert captured.get("nickname") == mask_nickname(MOCK_AUTHOR_NAME)
assert captured["nickname"] != MOCK_AUTHOR_NAME
assert "*" in captured["nickname"]
# 内容字段保留(video_id / desc / title)
assert captured["video_id"] == MOCK_VIDEO_ID
assert captured["desc"] == MOCK_CAPTION
assert captured["title"] == MOCK_CAPTION
# video_type 来自 video_item.type，被 str() 化
assert captured["video_type"] == "1"
# 计数字段被 str() 化
assert captured["liked_count"] == "12345"
assert captured["viewd_count"] == "67890"
def test_kuaishou_comment_v2_masks_user_info():
"""评论 V2(snake_case)格式：author_id/author_name 经匿名+脱敏，headurl 不落库。"""
with _patch_create_store() as holder:
asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_v2()))
captured = holder.comment
assert captured, "FakeStore 未捕获到 comment_item"
_assert_no_forbidden_keys(captured, "kuaishou_comment_v2")
assert "headurl" not in captured and "avatar" not in captured
# comment_id 由 int 转为 str
assert captured["comment_id"] == "9001"
assert captured["video_id"] == MOCK_VIDEO_ID
assert captured["content"] == "太搞笑了哈哈哈"
# V2 用 commentCount
assert captured["sub_comment_count"] == "7"
# creator_hash / 昵称
assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_V2_AUTHOR_ID)
assert captured["creator_hash"] != MOCK_COMMENT_V2_AUTHOR_ID
assert captured["nickname"] == mask_nickname(MOCK_COMMENT_V2_AUTHOR_NAME)
assert captured["nickname"] != MOCK_COMMENT_V2_AUTHOR_NAME
assert "*" in captured["nickname"]
def test_kuaishou_comment_legacy_masks_user_info():
"""评论旧 GraphQL(camelCase)格式：authorId/authorName 经匿名+脱敏，headurl 不落库。"""
with _patch_create_store() as holder:
asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_legacy()))
captured = holder.comment
assert captured, "FakeStore 未捕获到 comment_item"
_assert_no_forbidden_keys(captured, "kuaishou_comment_legacy")
assert "headurl" not in captured and "avatar" not in captured
# commentId 由 int 转为 str
assert captured["comment_id"] == "8001"
assert captured["video_id"] == MOCK_VIDEO_ID
assert captured["content"] == "这条评论来自旧 GraphQL 接口"
# 旧格式用 subCommentCount
assert captured["sub_comment_count"] == "3"
# creator_hash / 昵称
assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_LEGACY_AUTHOR_ID)
assert captured["creator_hash"] != MOCK_COMMENT_LEGACY_AUTHOR_ID
assert captured["nickname"] == mask_nickname(MOCK_COMMENT_LEGACY_AUTHOR_NAME)
assert captured["nickname"] != MOCK_COMMENT_LEGACY_AUTHOR_NAME
assert "*" in captured["nickname"]
def test_kuaishou_store_end_to_end_sqlite():
"""端到端：FakeStore 捕获真实 dict -> KuaishouVideo(**captured) ORM 列校验
-> 写入内存 SQLite -> 查询回读，验证属性正确且无禁用列。
全程在同一个事件循环内完成(aiosqlite 连接绑定事件循环，跨 loop 会报错)，
不 patch db_session.get_session，而是直接用自建内存 engine 走 ORM 写入/查询。"""
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
from database.models import Base, KuaishouVideo, KuaishouVideoComment
async def run():
# 内存 SQLite + StaticPool：单连接共享，保证 create_all 与后续读写同一库
engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool)
SessionFactory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
async with engine.begin() as conn:
await conn.run_sync(
Base.metadata.create_all,
tables=[KuaishouVideo.__table__, KuaishouVideoComment.__table__],
)
# 1) 经 update_kuaishou_video 产生真实存储 dict(FakeStore 捕获)
with _patch_create_store() as holder:
await update_kuaishou_video(make_mock_video())
captured = holder.content
assert captured, "FakeStore 未捕获到 content_item"
# 2) ORM 列校验：captured 的所有 key 必须是 KuaishouVideo 的合法列，
# 否则 KuaishouVideo(**captured) 抛 TypeError(若有禁用/多余键即暴露源码 bug)
valid_cols = {c.name for c in KuaishouVideo.__table__.columns}
assert set(captured.keys()) <= valid_cols, (
f"captured 含非合法列: {set(captured.keys()) - valid_cols}"
)
obj = KuaishouVideo(**captured) # 不抛异常即通过列校验
assert obj.video_id == MOCK_VIDEO_ID
assert obj.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID)
assert obj.creator_hash != MOCK_AUTHOR_ID
assert obj.nickname == mask_nickname(MOCK_AUTHOR_NAME)
assert obj.nickname != MOCK_AUTHOR_NAME
assert obj.desc == MOCK_CAPTION # 内容保留
# 3) 真实写库 + 查询回读
async with SessionFactory() as session:
session.add(obj)
await session.commit()
res = await session.execute(
select(KuaishouVideo).where(KuaishouVideo.video_id == MOCK_VIDEO_ID)
)
row = res.scalar_one()
assert row is not None
assert row.video_id == MOCK_VIDEO_ID
assert row.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID)
assert row.creator_hash != MOCK_AUTHOR_ID
assert row.nickname == mask_nickname(MOCK_AUTHOR_NAME)
assert row.nickname != MOCK_AUTHOR_NAME
# 内容字段保留
assert row.desc == MOCK_CAPTION
assert row.title == MOCK_CAPTION
# ORM 行对象上不应存在任何禁用列属性
for bad in FORBIDDEN_KEYS:
assert not hasattr(row, bad), f"KuaishouVideo 行对象仍含禁用属性: {bad}"
await engine.dispose()
asyncio.run(run())
if __name__ == "__main__":
pytest.main([__file__, "-v"])

239
tests/test_no_user_info.py Normal file
View File

@@ -0,0 +1,239 @@
# -*- coding: utf-8 -*-
"""
教学版回归测试：确保爬取/存储链路不再持久化可定位真人的用户个人信息。
覆盖：
1. ORM 自省 —— database.models 中无禁用列、creator 档案表已删除、内容/评论表含 creator_hash。
2. 提取层 —— 用 mock API/HTML payload 喂各平台提取器，断言输出 dict 不含禁用字段、
不含原始 user_id、昵称已脱敏且不等于原文。
3. 仓库 grep 断言 —— store/ 与 media_platform/ 不再把禁用字段作为存储 dict 的 key。
"""
import re
import subprocess
import pathlib
import pytest
ROOT = pathlib.Path(__file__).resolve().parent.parent
# 统一的禁用字段名(键)。昵称字段(nickname/user_nickname/screen_name/name/user_name)允许保留(值需脱敏)。
FORBIDDEN_KEYS = {
"user_id", "sec_uid", "short_user_id", "user_unique_id", "user_signature",
"avatar", "user_avatar", "face", "sign", "profile_url", "user_link",
"url_token", "user_url_token", "ip_location", "ip_address", "gender", "sex",
"up_id", "fan_id", "up_name", "fan_name", "up_avatar", "fan_avatar",
"up_sign", "fan_sign", "mid",
}
NICK_KEYS = {"nickname", "user_nickname", "screen_name", "name", "user_name"}
MASK_RE = re.compile(r"^.?\*{1,4}.?$")
# ----------------------------- ORM 自省 -----------------------------
def test_orm_has_no_forbidden_columns():
import database.models as m
from sqlalchemy.orm import class_mapper
tables = [c for c in dir(m) if c[0].isupper()
and c not in ("Base", "Column", "Integer", "BigInteger", "String", "Text")]
bad = []
for t in tables:
cols = {c.name for c in class_mapper(getattr(m, t)).columns}
hit = cols & FORBIDDEN_KEYS
if hit:
bad.append((t, sorted(hit)))
assert not bad, f"ORM 仍含禁用列: {bad}"
def test_creator_tables_removed():
import database.models as m
removed = {"XhsCreator", "DyCreator", "WeiboCreator", "TiebaCreator",
"ZhihuCreator", "BilibiliUpInfo", "BilibiliContactInfo"}
for t in removed:
assert not hasattr(m, t), f"creator 档案表 {t} 仍存在"
def test_content_tables_have_creator_hash():
import database.models as m
from sqlalchemy.orm import class_mapper
content_tables = ["XhsNote", "XhsNoteComment", "WeiboNote", "WeiboNoteComment",
"BilibiliVideo", "BilibiliVideoComment", "BilibiliUpDynamic",
"DouyinAweme", "DouyinAwemeComment", "KuaishouVideo",
"KuaishouVideoComment", "TiebaNote", "TiebaComment",
"ZhihuContent", "ZhihuComment"]
for t in content_tables:
cols = {c.name for c in class_mapper(getattr(m, t)).columns}
assert "creator_hash" in cols, f"{t} 缺少 creator_hash 列"
# ----------------------------- 提取层 mock -----------------------------
def _check_no_forbidden_keys(d: dict, label: str):
keys = set(d.keys())
hit = keys & FORBIDDEN_KEYS
assert not hit, f"[{label}] 输出仍含禁用字段键: {hit}"
def _check_nickname_masked(d: dict, raw: str, label: str):
nick_keys = set(d.keys()) & NICK_KEYS
assert nick_keys, f"[{label}] 未保留任何昵称字段(应保留并脱敏)"
for k in nick_keys:
val = d[k]
if val in ("", None):
continue
assert val != raw, f"[{label}] {k} 未脱敏，仍为原文: {val}"
assert MASK_RE.match(val) or "*" in val, f"[{label}] {k} 未脱敏: {val}"
def test_mask_and_hash_tools():
from tools.user_hash import anonymize_user_id, mask_nickname
h = anonymize_user_id("12345")
assert h and h != "12345" and re.fullmatch(r"[0-9a-f]{16}", h)
assert anonymize_user_id(None) == "" and anonymize_user_id("") == ""
# 昵称脱敏：首尾留1字、中间星号，且不等于原文
assert mask_nickname("张三丰") != "张三丰"
assert "*" in mask_nickname("张三丰")
assert mask_nickname(None) == ""
assert mask_nickname("a") == "*"
def test_xhs_note_extraction_masks_user_info():
import asyncio
import store.xhs as xs
note_item = {
"note_id": "abc",
"type": "normal",
"title": "t",
"desc": "d",
"time": 1,
"last_update_time": 0,
"user": {"user_id": "u123", "nickname": "小红同学", "avatar": "http://x/a.jpg"},
"ip_location": "上海",
"interact_info": {"liked_count": "1", "collected_count": "0",
"comment_count": "0", "share_count": "0"},
"image_list": [], "tag_list": [], "xsec_token": "tok",
}
captured = {}
class FakeStore:
async def store_content(self, content_item):
captured.update(content_item)
orig = xs.XhsStoreFactory.create_store
xs.XhsStoreFactory.create_store = staticmethod(lambda: FakeStore())
try:
asyncio.run(xs.update_xhs_note(note_item))
finally:
xs.XhsStoreFactory.create_store = orig
_check_no_forbidden_keys(captured, "xhs_note")
assert captured.get("creator_hash") != "u123"
_check_nickname_masked(captured, "小红同学", "xhs_note")
def test_tieba_note_extraction_masks_user_info():
from media_platform.tieba.help import TieBaExtractor
api_data = {
"thread": {"id": 1, "title": "tt", "reply_num": 5},
"first_floor": {"tid": 1, "author_id": 9, "time": 1700000000, "content": "c"},
"forum": {"name": "test", "id": 1},
"page": {"total_page": 1},
"user_list": [{"id": 9, "name_show": "贴吧老哥", "name": "lg", "portrait": "p", "ip_address": "北京"}],
}
note = TieBaExtractor().extract_note_detail_from_api(api_data)
d = note.model_dump()
_check_no_forbidden_keys(d, "tieba_note")
assert d.get("creator_hash") # user_link 已转哈希
_check_nickname_masked(d, "贴吧老哥", "tieba_note")
def test_tieba_comment_extraction_masks_user_info():
from media_platform.tieba.help import TieBaExtractor
from model.m_baidu_tieba import TiebaNote
api_data = {
"forum": {"id": 1, "name": "test"},
"post_list": [{"id": 7, "author_id": 9, "time": 1700000000, "content": "c", "sub_post_number": 0}],
"user_list": [{"id": 9, "name_show": "评论员", "name": "py", "portrait": "p", "ip_address": "上海"}],
}
note_detail = TiebaNote(note_id="1", title="t", note_url="u", tieba_name="test", tieba_link="l")
comments = TieBaExtractor().extract_tieba_note_parent_comments_from_api(api_data, note_detail)
assert comments
d = comments[0].model_dump()
_check_no_forbidden_keys(d, "tieba_comment")
_check_nickname_masked(d, "评论员", "tieba_comment")
def test_zhihu_comment_extraction_masks_user_info():
from media_platform.zhihu.help import ZhihuExtractor
from model.m_zhihu import ZhihuContent
comments_raw = [{
"type": "comment", "id": 1, "content": "c", "created_time": 1700000000,
"like_count": 1, "dislike_count": 0, "child_comment_count": 0,
"author": {"id": "z9", "name": "知乎答主", "url_token": "tok", "avatar_url": "http://x/a.jpg"},
"comment_tag": [{"type": "ip_info", "text": "广东"}],
}]
page_content = ZhihuContent(content_id="c1", content_type="answer")
comments = ZhihuExtractor().extract_comments(page_content, comments_raw)
assert comments
d = comments[0].model_dump() if hasattr(comments[0], "model_dump") else vars(comments[0])
_check_no_forbidden_keys(d, "zhihu_comment")
assert "creator_hash" in d and d["creator_hash"]
_check_nickname_masked(d, "知乎答主", "zhihu_comment")
def test_bilibili_video_dict_masks_user_info():
# 直接测 store/bilibili/__init__.py 的拍平逻辑(不触发网络)
import asyncio
from store.bilibili import update_bilibili_video
video_item = {
"View": {
"aid": 100, "title": "t", "desc": "d", "pubdate": 1,
"owner": {"mid": 777, "name": "UP主大人", "face": "http://x/a.jpg"},
"stat": {"like": 1, "view": 2},
"pic": "http://x/cover.jpg",
}
}
# 拦截真实存储：替换工厂返回一个捕获 dict 的假 store
captured = {}
class FakeStore:
async def store_content(self, content_item):
captured.update(content_item)
import store.bilibili as bs
orig = bs.BiliStoreFactory.create_store
bs.BiliStoreFactory.create_store = staticmethod(lambda: FakeStore())
try:
asyncio.get_event_loop().run_until_complete(update_bilibili_video(video_item)) \
if False else asyncio.run(update_bilibili_video(video_item))
finally:
bs.BiliStoreFactory.create_store = orig
_check_no_forbidden_keys(captured, "bili_video")
assert captured.get("creator_hash") != 777
_check_nickname_masked(captured, "UP主大人", "bili_video")
# ----------------------------- 仓库 grep 断言 -----------------------------
def test_store_no_forbidden_dict_keys():
# store/ 下不得把禁用字段作为存储 dict 的 key("field": value 形式)
out = subprocess.run(
["grep", "-rnE", '"(' + "|".join(FORBIDDEN_KEYS) + r')"\s*:', str(ROOT / "store")],
capture_output=True, text=True,
)
# 允许的例外：Mongo store_creator 里的 query={"user_id": ...} 已全部改为 pass，应为空
assert out.stdout.strip() == "", f"store/ 仍写入禁用字段键:\n{out.stdout}"
def test_store_no_creator_orm_imports():
# 已删除的 creator ORM 表(XhsCreator/DyCreator/...)不得再从 database.models 导入。
# 注意:model/m_*.py 里的同名 pydantic 类是内存类型，允许保留。
out = subprocess.run(
["grep", "-rnE",
r"from database\.models import.*(XhsCreator|DyCreator|WeiboCreator|TiebaCreator|ZhihuCreator|BilibiliUpInfo|BilibiliContactInfo)",
str(ROOT / "store")],
capture_output=True, text=True,
)
assert out.stdout.strip() == "", f"store/ 仍 import 已删除的 creator ORM 表:\n{out.stdout}"
if __name__ == "__main__":
pytest.main([__file__, "-v"])

278
tests/test_weibo_no_user_info.py Normal file
View File

@@ -0,0 +1,278 @@
# -*- coding: utf-8 -*-
"""
教学版回归测试(微博 weibo):确保微博存储链路不再持久化可定位真人的用户个人信息。
覆盖:
1. test_weibo_note_masks_user_info —— 用贴近真实微博结构的 mock note_item 喂
store.weibo.update_weibo_note,用 FakeStore 捕获拍平后的存储 dict,断言:
- 不含任何禁用字段键(user_id/avatar/gender/profile_url/ip_location/desc ...)
- 含 creator_hash,且不等于原始 user id
- nickname 已脱敏且不等于原文
2. test_weibo_comment_masks_user_info —— 同上,对 update_weibo_note_comment。
3. test_weibo_store_end_to_end_sqlite —— 端到端:把 update_weibo_note /
update_weibo_note_comment 产生的真实 dict 用 SQLite 内存库走完整 ORM
写入+查询。WeiboNote(**captured_dict) 会因 SQLAlchemy 声明式构造器对未知
关键字的校验,在 dict 含已删列时直接抛 TypeError —— 以此证明 dict 的 key
与删列后的 ORM 完全对得上,且表中无禁用列、有 creator_hash。
说明:微博 note 的正文存于 content 字段,update_weibo_note 不产生 desc 键
(WeiboNote ORM 亦无 desc 列),故不存在用户 description 被持久化的风险。
"""
import asyncio
import pytest
# 原始(明文)测试数据
RAW_USER_ID = 7654321
RAW_NICKNAME = "微博达人"
RAW_COMMENT_USER_ID = 111222
RAW_COMMENT_NICKNAME = "评论员小张"
NOTE_ID = 5123456789
COMMENT_ID = 998877
# 合法 RFC2822 时间串(weekday 与日期已对齐:2025-06-14 是周六)
RFC2822_TIME = "Sat Jun 14 12:00:00 +0800 2025"
# 禁用字段名(键)。昵称字段允许保留,但值必须脱敏。
FORBIDDEN_KEYS = {
"user_id", "sec_uid", "short_user_id", "user_unique_id",
"avatar", "user_avatar", "face", "sign", "profile_url", "user_link",
"ip_location", "ip_address", "gender", "sex", "desc",
}
# ----------------------------- mock 数据 -----------------------------
def make_mock_note() -> dict:
"""贴近真实 m.weibo.cn 接口结构的 mock note_item(含嵌套 user 信息)。"""
return {
"mblog": {
"id": NOTE_ID,
"text": "今天天气不错 <a href='#'>@好友</a> 出去玩",
"created_at": RFC2822_TIME,
"attitudes_count": 10,
"comments_count": 2,
"reposts_count": 1,
"user": {
"id": RAW_USER_ID,
"screen_name": RAW_NICKNAME,
"avatar_hd": "https://wx avatar.example.com/7654321.jpg",
"gender": "f",
"profile_url": "https://m.weibo.cn/profile/7654321",
"description": "这是一个用户签名",
"ip_location": "上海",
"followers_count": 9999,
},
}
}
def make_mock_comment() -> dict:
"""贴近真实微博评论结构的 mock comment_item(含嵌套 user 信息)。"""
return {
"id": COMMENT_ID,
"text": "说得好 <a href='#'>支持</a>",
"created_at": RFC2822_TIME,
"total_number": 3,
"like_count": 5,
"rootid": "parent_abc",
"user": {
"id": RAW_COMMENT_USER_ID,
"screen_name": RAW_COMMENT_NICKNAME,
"avatar_hd": "https://wx avatar.example.com/111222.jpg",
"gender": "m",
"profile_url": "https://m.weibo.cn/profile/111222",
"description": "评论员签名",
"ip_location": "广东",
},
}
# ----------------------------- FakeStore 捕获 -----------------------------
class _FakeStore:
"""捕获 store_content / store_comment 收到的 dict,不触发任何真实存储。"""
def __init__(self):
self.captured_content = {}
self.captured_comment = {}
async def store_content(self, content_item):
self.captured_content.update(content_item)
async def store_comment(self, comment_item):
self.captured_comment.update(comment_item)
async def store_creator(self, creator):
pass
def _patch_factory(fake: "_FakeStore"):
"""把 store.weibo.WeibostoreFactory.create_store 替换为返回 fake 的静态方法,
返回 (module, orig) 便于 finally 还原。"""
import store.weibo as wb
orig = wb.WeibostoreFactory.create_store
wb.WeibostoreFactory.create_store = staticmethod(lambda: fake)
return wb, orig
def _restore(wb, orig):
wb.WeibostoreFactory.create_store = orig
# ----------------------------- 测试 -----------------------------
def test_weibo_note_masks_user_info():
"""note 拍平后的存储 dict 不含禁用键、creator_hash 不等于原始 user id、昵称已脱敏。"""
import store.weibo as wb
fake = _FakeStore()
wb_, orig = _patch_factory(fake)
try:
asyncio.run(wb.update_weibo_note(make_mock_note()))
finally:
_restore(wb_, orig)
captured = fake.captured_content
assert captured, "FakeStore 未捕获到 note dict"
# 1. 不含任何禁用字段键
hit = set(captured.keys()) & FORBIDDEN_KEYS
assert not hit, f"note 存储 dict 仍含禁用字段键: {hit}"
# 2. creator_hash 存在、是 16 位 hex、不等于原始 user id
creator_hash = captured.get("creator_hash")
assert creator_hash, "note dict 缺少 creator_hash"
assert creator_hash != str(RAW_USER_ID)
assert creator_hash != RAW_USER_ID
assert len(creator_hash) == 16
# 3. 昵称已脱敏:不等于原文且含星号
nickname = captured.get("nickname")
assert nickname, "note dict 缺少 nickname"
assert nickname != RAW_NICKNAME, "note 昵称未脱敏,仍为原文"
assert "*" in nickname, f"note 昵称未脱敏: {nickname}"
# 4. 内容字段正确(正文存于 content,不是 desc)
assert "hello" not in captured # 确认没误存
assert "今天天气不错" in captured["content"]
assert captured["note_id"] == NOTE_ID
assert captured["liked_count"] == "10"
assert captured["comments_count"] == "2"
assert captured["shared_count"] == "1"
def test_weibo_comment_masks_user_info():
"""comment 拍平后的存储 dict 不含禁用键、creator_hash 不等于原始 user id、昵称已脱敏。"""
import store.weibo as wb
fake = _FakeStore()
wb_, orig = _patch_factory(fake)
try:
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
finally:
_restore(wb_, orig)
captured = fake.captured_comment
assert captured, "FakeStore 未捕获到 comment dict"
# 1. 不含任何禁用字段键
hit = set(captured.keys()) & FORBIDDEN_KEYS
assert not hit, f"comment 存储字典仍含禁用字段键: {hit}"
# 2. creator_hash 存在、不等于原始 user id
creator_hash = captured.get("creator_hash")
assert creator_hash, "comment dict 缺少 creator_hash"
assert creator_hash != str(RAW_COMMENT_USER_ID)
assert creator_hash != RAW_COMMENT_USER_ID
assert len(creator_hash) == 16
# 3. 昵称已脱敏
nickname = captured.get("nickname")
assert nickname, "comment dict 缺少 nickname"
assert nickname != RAW_COMMENT_NICKNAME, "comment 昵称未脱敏,仍为原文"
assert "*" in nickname, f"comment 昵称未脱敏: {nickname}"
# 4. 内容字段正确
assert "说得好" in captured["content"]
assert captured["comment_id"] == str(COMMENT_ID)
assert captured["comment_like_count"] == "5"
assert captured["sub_comment_count"] == "3"
assert captured["parent_comment_id"] == "parent_abc"
def test_weibo_store_end_to_end_sqlite():
"""端到端:捕获 note/comment 的真实 dict,用 SQLite 内存库走完整 ORM 写入+查询。
关键点:WeiboNote(**captured_dict) / WeiboNoteComment(**captured_dict) 会触发
SQLAlchemy 声明式构造器的关键字校验——若 dict 含已删列(如 avatar/gender)会直接
抛 TypeError。此处不抛异常即证明 dict 的 key 与删列后的 ORM 列完全对得上。
"""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import store.weibo as wb
from database.models import Base, WeiboNote, WeiboNoteComment
# ---- 1. 用 FakeStore 捕获 update_weibo_note / update_weibo_note_comment 产生的真实 dict ----
fake = _FakeStore()
wb_, orig = _patch_factory(fake)
try:
asyncio.run(wb.update_weibo_note(make_mock_note()))
asyncio.run(wb.update_weibo_note_comment(str(NOTE_ID), make_mock_comment()))
finally:
_restore(wb_, orig)
captured_note = dict(fake.captured_content)
captured_comment = dict(fake.captured_comment)
assert captured_note and captured_comment
# ---- 2. SQLite 内存库,建 weibo 两张表 ----
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine, tables=[WeiboNote.__table__, WeiboNoteComment.__table__])
Session = sessionmaker(bind=engine)
session = Session()
try:
# ---- 3. note:构造 ORM 对象(dict 多了已删列会直接 TypeError)并写入 ----
note_obj = WeiboNote(**captured_note) # 不抛异常 => key 与 ORM 列对得上
session.add(note_obj)
session.commit()
row = session.query(WeiboNote).one()
note_cols = {c.name for c in WeiboNote.__table__.columns}
# 表结构层面无禁用列
assert not (note_cols & FORBIDDEN_KEYS), \
f"WeiboNote 表仍含禁用列: {note_cols & FORBIDDEN_KEYS}"
# 行数据层面:creator_hash 正确、昵称脱敏、正文保留
assert row.creator_hash and row.creator_hash != str(RAW_USER_ID)
assert row.nickname != RAW_NICKNAME and "*" in row.nickname
assert row.note_id == NOTE_ID
assert "今天天气不错" in row.content
# 确认没有 desc 列存任何用户描述
assert "desc" not in note_cols
# ---- 4. comment:同上。注意 store_comment 实现会把 comment_id/note_id/create_time
# 转成 int 后再构造 ORM,此处忠实复刻该转换以走通 BigInteger 列写入 ----
cc = dict(captured_comment)
cc["comment_id"] = int(cc["comment_id"])
cc["note_id"] = int(cc.get("note_id", 0) or 0)
cc["create_time"] = int(cc.get("create_time", 0) or 0)
comment_obj = WeiboNoteComment(**cc) # 不抛异常 => key 与 ORM 列对得上
session.add(comment_obj)
session.commit()
crow = session.query(WeiboNoteComment).one()
comment_cols = {c.name for c in WeiboNoteComment.__table__.columns}
assert not (comment_cols & FORBIDDEN_KEYS), \
f"WeiboNoteComment 表仍含禁用列: {comment_cols & FORBIDDEN_KEYS}"
assert crow.creator_hash and crow.creator_hash != str(RAW_COMMENT_USER_ID)
assert crow.nickname != RAW_COMMENT_NICKNAME and "*" in crow.nickname
assert crow.comment_id == COMMENT_ID
assert crow.note_id == NOTE_ID
assert "说得好" in crow.content
finally:
session.close()
engine.dispose()
if __name__ == "__main__":
pytest.main([__file__, "-v"])

36
tools/user_hash.py Normal file
View File

@@ -0,0 +1,36 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# 本文件为 MediaCrawler 教学版的一部分。
# 出于教学与防骚扰定位，爬取结果中不保留任何可定位到真人的用户个人信息
# （用户 ID、IP 归属地、头像、主页链接、签名、性别等一律不采集；
# 昵称保留但做中间脱敏）。本模块提供匿名化与脱敏工具。
import hashlib
def anonymize_user_id(user_id) -> str:
"""把原始用户 ID 转成匿名哈希，用于内容/评论记录的创作者分组，
不暴露真实身份。返回 sha256 截断 16 位的十六进制串。"""
if user_id is None:
return ""
s = str(user_id).strip()
if not s:
return ""
return hashlib.sha256(s.encode("utf-8")).hexdigest()[:16]
def mask_nickname(name) -> str:
"""昵称中间脱敏：首尾各保留 1 字，中间替换为星号。
- 长度 <= 1：返回 "*"
- 长度 == 2：首字 + "*"
- 长度 >= 3：首字 + "***" + 尾字
这样既保留教学分析所需的内容归属语义，又无法据昵称定位到真人。
"""
if name is None:
return ""
s = str(name)
if len(s) <= 1:
return "*"
if len(s) == 2:
return s[0] + "*"
return s[0] + "***" + s[-1]