refactor: 教学版移除全平台用户个人信息采集与持久化

- 用户 ID 转为匿名 creator_hash，昵称中间脱敏，IP/头像/主页/签名/性别不再采集
- 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台
- 删除 7 张 creator 档案 ORM 表，15 张内容/评论表新增 creator_hash 列
- B 站禁用粉丝/关注/联系人列表抓取
- 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试

测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed)

media_platform/bilibili/core.py

@@ -646,14 +646,14 @@ class BilibiliCrawler(AbstractCrawler):
         """
         async with semaphore:
             creator_unhandled_info: Dict = await self.bili_client.get_creator_info(creator_id)
+            # 教学版：仅保留动态所需的最少字段(内存临时用)，不持久化创作者个人资料。
             creator_info: Dict = {
                 "id": creator_id,
                 "name": creator_unhandled_info.get("name"),
-                "sign": creator_unhandled_info.get("sign"),
-                "avatar": creator_unhandled_info.get("face"),
             }
-        await self.get_fans(creator_info, semaphore)
-        await self.get_followings(creator_info, semaphore)
+        # 教学版：不再爬取粉丝/关注列表(其他用户的个人信息)，防骚扰。
+        # await self.get_fans(creator_info, semaphore)
+        # await self.get_followings(creator_info, semaphore)
         await self.get_dynamics(creator_info, semaphore)
 
     async def get_fans(self, creator_info: Dict, semaphore: asyncio.Semaphore):

media_platform/tieba/help.py

@@ -30,6 +30,7 @@ from parsel import Selector
 from constant import baidu_tieba as const
 from model.m_baidu_tieba import TiebaComment, TiebaCreator, TiebaNote
 from tools import utils
+from tools.user_hash import anonymize_user_id, mask_nickname
 
 GENDER_MALE = "sex_male"
 GENDER_FEMALE = "sex_female"
@@ -143,9 +144,8 @@ class TieBaExtractor:
                 publish_time=utils.get_time_str_from_unix_time(
                     item.get("time") or item.get("create_time") or 0
                 ),
-                user_link="",
-                user_nickname=user.get("show_nickname") or user.get("user_name") or "",
-                user_avatar=user.get("portrait") or user.get("portraith") or "",
+                creator_hash=anonymize_user_id(user.get("id") or user.get("portrait") or ""),
+                user_nickname=mask_nickname(user.get("show_nickname") or user.get("user_name") or ""),
                 tieba_name=tieba_name,
                 tieba_link=self._tieba_link_from_name(tieba_name),
                 total_replay_num=item.get("post_num") or 0,
@@ -177,14 +177,12 @@ class TieBaExtractor:
             publish_time=utils.get_time_str_from_unix_time(
                 first_floor.get("time") or thread.get("create_time") or 0
             ),
-            user_link=self._api_user_link(author),
-            user_nickname=author.get("name_show") or author.get("name") or "",
-            user_avatar=self._api_user_avatar(author),
+            creator_hash=anonymize_user_id(self._api_user_link(author)),
+            user_nickname=mask_nickname(author.get("name_show") or author.get("name") or ""),
             tieba_name=tieba_name,
             tieba_link=self._tieba_link_from_name(tieba_name),
             total_replay_num=thread.get("reply_num") or 0,
             total_replay_page=page.get("total_page") or 0,
-            ip_location=author.get("ip_address") or "",
         )
         return note
 
@@ -210,13 +208,11 @@ class TieBaExtractor:
                 sub_comment_count=item.get("sub_post_number") or 0,
                 content=self._extract_api_content_text(item.get("content")),
                 note_url=note_detail.note_url,
-                user_link=self._api_user_link(user),
-                user_nickname=user.get("name_show") or user.get("name") or "",
-                user_avatar=self._api_user_avatar(user),
+                creator_hash=anonymize_user_id(self._api_user_link(user)),
+                user_nickname=mask_nickname(user.get("name_show") or user.get("name") or ""),
                 tieba_id=tieba_id,
                 tieba_name=tieba_name,
                 tieba_link=tieba_link,
-                ip_location=user.get("ip_address") or "",
                 publish_time=utils.get_time_str_from_unix_time(item.get("time") or 0),
                 note_id=note_detail.note_id,
             )
@@ -230,20 +226,11 @@ class TieBaExtractor:
         user = api_data.get("data", {}).get("user", {})
         if not user:
             raise ValueError(f"Creator API response does not contain user info: {api_data}")
-        gender_value = user.get("sex", user.get("gender", 0))
-        gender = "Unknown"
-        if gender_value == 1:
-            gender = "Male"
-        elif gender_value == 2:
-            gender = "Female"
 
+        # 教学版：创作者个人资料不再落库，仅保留匿名哈希与脱敏昵称作内存对象。
         return TiebaCreator(
-            user_id=str(user.get("id", "")),
-            user_name=str(user.get("name", "")),
-            nickname=str(user.get("name_show") or user.get("name") or ""),
-            avatar=self._api_user_avatar(user),
-            gender=gender,
-            ip_location=str(user.get("ip_address", "")),
+            creator_hash=anonymize_user_id(str(user.get("id", ""))),
+            user_nickname=mask_nickname(str(user.get("name_show") or user.get("name") or "")),
             follows=int(user.get("concern_num") or 0),
             fans=int(user.get("fans_num") or 0),
             registration_duration=str(user.get("tb_age", "")),
@@ -370,10 +357,10 @@ class TieBaExtractor:
                     post, f".//div[{extractor._class_contains('p_content')}]"
                 ),
                 note_url=note_url,
-                user_nickname=extractor._selector_text(
+                creator_hash=anonymize_user_id(extractor._absolute_url(user_selector.xpath("./@href").get(default=""))),
+                user_nickname=mask_nickname(extractor._selector_text(
                     post, ".//a[contains(@href, '/home/main')][1]"
-                ),
-                user_link=extractor._absolute_url(user_selector.xpath("./@href").get(default="")),
+                )),
                 tieba_name=extractor._selector_text(
                     post, f".//a[{extractor._class_contains('p_forum')}][1]"
                 ),
@@ -451,8 +438,8 @@ class TieBaExtractor:
                 title=title,
                 desc=desc,
                 note_url=f"{const.TIEBA_URL}/p/{note_id}",
-                user_nickname=user_nickname,
-                user_link="",
+                creator_hash="",
+                user_nickname=mask_nickname(user_nickname),
                 tieba_name=tieba_name,
                 tieba_link=tieba_link,
                 publish_time=publish_time,
@@ -499,8 +486,8 @@ class TieBaExtractor:
                     post_selector, f".//div[{self._class_contains('threadlist_abs')}]"
                 ),
                 note_url=const.TIEBA_URL + f"/p/{note_id}",
-                user_link=self._absolute_url(user_selector.xpath("./@href").get(default="")),
-                user_nickname=user_nickname,
+                creator_hash=anonymize_user_id(self._absolute_url(user_selector.xpath("./@href").get(default=""))),
+                user_nickname=mask_nickname(user_nickname),
                 tieba_name=tieba_name,
                 tieba_link=tieba_link,
                 total_replay_num=post_field_value.get("reply_num", 0),
@@ -548,18 +535,14 @@ class TieBaExtractor:
             title=content_selector.xpath("//title/text()").get(default="").strip(),
             desc=content_selector.xpath("//meta[@name='description']/@content").get(default="").strip(),
             note_url=const.TIEBA_URL + f"/p/{note_id}",
-            user_link=self._absolute_url(author_link),
-            user_nickname=(
+            creator_hash=anonymize_user_id(self._absolute_url(author_link)),
+            user_nickname=mask_nickname(
                 self._selector_text(first_floor_selector, f".//a[{self._class_contains('p_author_name')}][1]")
                 or author_value.get("user_nickname")
                 or author_value.get("user_name", "")
             ),
-            user_avatar=first_floor_selector.xpath(
-                f".//a[{self._class_contains('p_author_face')}]//img/@src"
-            ).get(default="").strip(),
             tieba_name=tieba_name,
             tieba_link=tieba_link,
-            ip_location=ip_location,
             publish_time=publish_time,
             total_replay_num=(
                 thread_num_infos[0].xpath("./text()").get(default="0").strip()
@@ -623,13 +606,11 @@ class TieBaExtractor:
                 sub_comment_count=comment_content_value.get("comment_num") or 0,
                 content=utils.extract_text_from_html(content_html),
                 note_url=const.TIEBA_URL + f"/p/{note_id}",
-                user_link=self._absolute_url(user_selector.xpath("./@href").get(default="")),
-                user_nickname=user_nickname,
-                user_avatar=user_avatar,
+                creator_hash=anonymize_user_id(self._absolute_url(user_selector.xpath("./@href").get(default=""))),
+                user_nickname=mask_nickname(user_nickname),
                 tieba_id=str(comment_content_value.get("forum_id", "")),
                 tieba_name=tieba_name,
                 tieba_link=tieba_link,
-                ip_location=ip_location,
                 publish_time=publish_time,
                 note_id=note_id,
             )
@@ -662,9 +643,8 @@ class TieBaExtractor:
                 comment_ele.xpath(f".//span[{self._class_contains('lzl_content_main')}]").get(default=""))
             comment = TiebaComment(
                 comment_id=str(comment_value.get("spid")), content=content,
-                user_link=self._absolute_url(comment_user_a_selector.xpath("./@href").get(default="")),
-                user_nickname=str(comment_value.get("showname") or ""),
-                user_avatar=comment_user_a_selector.xpath("./img/@src").get(default=""),
+                creator_hash=anonymize_user_id(self._absolute_url(comment_user_a_selector.xpath("./@href").get(default=""))),
+                user_nickname=mask_nickname(str(comment_value.get("showname") or "")),
                 publish_time=self._selector_text(comment_ele, f".//span[{self._class_contains('lzl_time')}]"),
                 parent_comment_id=parent_comment.comment_id,
                 note_id=parent_comment.note_id, note_url=parent_comment.note_url,
@@ -695,13 +675,13 @@ class TieBaExtractor:
         if len(follow_fans_selector) == 2:
             follows, fans = self.extract_follow_and_fans(follow_fans_selector)
         user_content = userinfo_userdata_selector.get(default='')
-        return TiebaCreator(user_id=user_id, user_name=user_name,
-                            nickname=selector.xpath(".//span[@class='userinfo_username ']/text()").get(
-                                default='').strip(),
-                            avatar=selector.xpath(".//div[@class='userinfo_left_head']//img/@src").get(
-                                default='').strip(),
-                            gender=self.extract_gender(user_content),
-                            ip_location=self.extract_ip(user_content),
+        # 教学版：创作者个人资料不再落库，仅保留匿名哈希与脱敏昵称作内存对象。
+        return TiebaCreator(creator_hash=anonymize_user_id(user_id or user_link),
+                            user_nickname=mask_nickname(
+                                selector.xpath(".//span[@class='userinfo_username ']/text()").get(
+                                    default='').strip()
+                                or user_name
+                            ),
                             follows=follows,
                             fans=fans,
                             registration_duration=self.extract_registration_duration(user_content)
@@ -860,8 +840,8 @@ def test_extract_tieba_note_sub_comments():
     with open("test_data/note_sub_comments.html", "r", encoding="utf-8") as f:
         content = f.read()
         extractor = TieBaExtractor()
-        fake_parment_comment = TiebaComment(comment_id="123456", content="content", user_link="user_link",
-                                            user_nickname="user_nickname", user_avatar="user_avatar",
+        fake_parment_comment = TiebaComment(comment_id="123456", content="content", creator_hash="creator_hash",
+                                            user_nickname="user_nickname",
                                             publish_time="publish_time", parent_comment_id="parent_comment_id",
                                             note_id="note_id", note_url="note_url", tieba_id="tieba_id",
                                             tieba_name="tieba_name", )

media_platform/zhihu/client.py

@@ -459,11 +459,11 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
         }
         return await self.get(uri, params)
 
-    async def get_all_anwser_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
+    async def get_all_anwser_by_creator(self, url_token: str, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
         """
         Get all answers by creator
         Args:
-            creator: Creator information
+            url_token: Creator url token (in-memory only, not persisted)
             crawl_interval: Crawl delay interval in seconds
             callback: Callback after completing one crawl
 
@@ -475,10 +475,10 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
         offset: int = 0
         limit: int = 20
         while not is_end:
-            res = await self.get_creator_answers(creator.url_token, offset, limit)
+            res = await self.get_creator_answers(url_token, offset, limit)
             if not res:
                 break
-            utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {creator.url_token} answers: {res}")
+            utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {url_token} answers: {res}")
             paging_info = res.get("paging", {})
             is_end = paging_info.get("is_end")
             contents = self._extractor.extract_content_list_from_creator(res.get("data"))
@@ -491,14 +491,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
 
     async def get_all_articles_by_creator(
         self,
-        creator: ZhihuCreator,
+        url_token: str,
         crawl_interval: float = 1.0,
         callback: Optional[Callable] = None,
     ) -> List[ZhihuContent]:
         """
         Get all articles by creator
         Args:
-            creator:
+            url_token: Creator url token (in-memory only, not persisted)
             crawl_interval:
             callback:
 
@@ -510,7 +510,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
         offset: int = 0
         limit: int = 20
         while not is_end:
-            res = await self.get_creator_articles(creator.url_token, offset, limit)
+            res = await self.get_creator_articles(url_token, offset, limit)
             if not res:
                 break
             paging_info = res.get("paging", {})
@@ -525,14 +525,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
 
     async def get_all_videos_by_creator(
         self,
-        creator: ZhihuCreator,
+        url_token: str,
         crawl_interval: float = 1.0,
         callback: Optional[Callable] = None,
     ) -> List[ZhihuContent]:
         """
         Get all videos by creator
         Args:
-            creator:
+            url_token: Creator url token (in-memory only, not persisted)
             crawl_interval:
             callback:
 
@@ -544,7 +544,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
         offset: int = 0
         limit: int = 20
         while not is_end:
-            res = await self.get_creator_videos(creator.url_token, offset, limit)
+            res = await self.get_creator_videos(url_token, offset, limit)
             if not res:
                 break
             paging_info = res.get("paging", {})

media_platform/zhihu/core.py

@@ -276,27 +276,26 @@ class ZhihuCrawler(AbstractCrawler):
             utils.logger.info(
                 f"[ZhihuCrawler.get_creators_and_notes] Creator info: {createor_info}"
             )
-            await zhihu_store.save_creator(creator=createor_info)
 
             # By default, only answer information is extracted, uncomment below if articles and videos are needed
 
             # Get all anwser information of the creator
             all_content_list = await self.zhihu_client.get_all_anwser_by_creator(
-                creator=createor_info,
+                url_token=user_url_token,
                 crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                 callback=zhihu_store.batch_update_zhihu_contents,
             )
 
             # Get all articles of the creator's contents
             # all_content_list = await self.zhihu_client.get_all_articles_by_creator(
-            #     creator=createor_info,
+            #     url_token=user_url_token,
             #     crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
             #     callback=zhihu_store.batch_update_zhihu_contents
             # )
 
             # Get all videos of the creator's contents
             # all_content_list = await self.zhihu_client.get_all_videos_by_creator(
-            #     creator=createor_info,
+            #     url_token=user_url_token,
             #     crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
             #     callback=zhihu_store.batch_update_zhihu_contents
             # )

media_platform/zhihu/help.py

@@ -30,6 +30,7 @@ from constant import zhihu as zhihu_constant
 from model.m_zhihu import ZhihuComment, ZhihuContent, ZhihuCreator
 from tools import utils
 from tools.crawler_util import extract_text_from_html
+from tools.user_hash import anonymize_user_id, mask_nickname
 
 ZHIHU_SGIN_JS = None
 
@@ -120,11 +121,8 @@ class ZhihuExtractor:
 
         # extract author info
         author_info = self._extract_content_or_comment_author(answer.get("author"))
-        res.user_id = author_info.user_id
-        res.user_link = author_info.user_link
+        res.creator_hash = author_info.creator_hash
         res.user_nickname = author_info.user_nickname
-        res.user_avatar = author_info.user_avatar
-        res.user_url_token = author_info.url_token
         return res
 
     def _extract_article_content(self, article: Dict) -> ZhihuContent:
@@ -150,11 +148,8 @@ class ZhihuExtractor:
 
         # extract author info
         author_info = self._extract_content_or_comment_author(article.get("author"))
-        res.user_id = author_info.user_id
-        res.user_link = author_info.user_link
+        res.creator_hash = author_info.creator_hash
         res.user_nickname = author_info.user_nickname
-        res.user_avatar = author_info.user_avatar
-        res.user_url_token = author_info.url_token
         return res
 
     def _extract_zvideo_content(self, zvideo: Dict) -> ZhihuContent:
@@ -184,11 +179,8 @@ class ZhihuExtractor:
 
         # extract author info
         author_info = self._extract_content_or_comment_author(zvideo.get("author"))
-        res.user_id = author_info.user_id
-        res.user_link = author_info.user_link
+        res.creator_hash = author_info.creator_hash
         res.user_nickname = author_info.user_nickname
-        res.user_avatar = author_info.user_avatar
-        res.user_url_token = author_info.url_token
         return res
 
     @staticmethod
@@ -207,11 +199,8 @@ class ZhihuExtractor:
                 return res
             if not author.get("id"):
                 author = author.get("member")
-            res.user_id = author.get("id")
-            res.user_link = f"{zhihu_constant.ZHIHU_URL}/people/{author.get('url_token')}"
-            res.user_nickname = author.get("name")
-            res.user_avatar = author.get("avatar_url")
-            res.url_token = author.get("url_token")
+            res.creator_hash = anonymize_user_id(author.get("id"))
+            res.user_nickname = mask_nickname(author.get("name"))
 
         except Exception as e :
             utils.logger.warning(
@@ -253,7 +242,6 @@ class ZhihuExtractor:
         res.parent_comment_id = comment.get("reply_comment_id")
         res.content = extract_text_from_html(comment.get("content"))
         res.publish_time = comment.get("created_time")
-        res.ip_location = self._extract_comment_ip_location(comment.get("comment_tag", []))
         res.sub_comment_count = comment.get("child_comment_count")
         res.like_count = comment.get("like_count") if comment.get("like_count") else 0
         res.dislike_count = comment.get("dislike_count") if comment.get("dislike_count") else 0
@@ -262,10 +250,8 @@ class ZhihuExtractor:
 
         # extract author info
         author_info = self._extract_content_or_comment_author(comment.get("author"))
-        res.user_id = author_info.user_id
-        res.user_link = author_info.user_link
+        res.creator_hash = author_info.creator_hash
         res.user_nickname = author_info.user_nickname
-        res.user_avatar = author_info.user_avatar
         return res
 
     @staticmethod
@@ -352,13 +338,8 @@ class ZhihuExtractor:
             return None
 
         res = ZhihuCreator()
-        res.user_id = creator_info.get("id")
-        res.user_link = f"{zhihu_constant.ZHIHU_URL}/people/{user_url_token}"
-        res.user_nickname = creator_info.get("name")
-        res.user_avatar = creator_info.get("avatarUrl")
-        res.url_token = creator_info.get("urlToken") or user_url_token
-        res.gender = self._foramt_gender_text(creator_info.get("gender"))
-        res.ip_location = creator_info.get("ipInfo")
+        res.creator_hash = anonymize_user_id(creator_info.get("id"))
+        res.user_nickname = mask_nickname(creator_info.get("name"))
         res.follows = creator_info.get("followingCount")
         res.fans = creator_info.get("followerCount")
         res.anwser_count = creator_info.get("answerCount")