mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-06 01:47:26 +08:00
i18n: translate all Chinese comments, docstrings, and logger messages to English
Comprehensive translation of Chinese text to English across the entire codebase: - api/: FastAPI server documentation and logger messages - cache/: Cache abstraction layer comments and docstrings - database/: Database models and MongoDB store documentation - media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu) - model/: Data model documentation - proxy/: Proxy pool and provider documentation - store/: Data storage layer comments - tools/: Utility functions and browser automation - test/: Test file documentation Preserved: Chinese disclaimer header (lines 10-18) for legal compliance 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -45,7 +45,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timeout=60, # 若开启爬取媒体选项,xhs 的长视频需要更久的超时时间
|
||||
timeout=60, # If media crawling is enabled, Xiaohongshu long videos need longer timeout
|
||||
proxy=None,
|
||||
*,
|
||||
headers: Dict[str, str],
|
||||
@@ -58,30 +58,30 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
self.headers = headers
|
||||
self._host = "https://edith.xiaohongshu.com"
|
||||
self._domain = "https://www.xiaohongshu.com"
|
||||
self.IP_ERROR_STR = "网络连接异常,请检查网络设置或重启试试"
|
||||
self.IP_ERROR_STR = "Network connection error, please check network settings or restart"
|
||||
self.IP_ERROR_CODE = 300012
|
||||
self.NOTE_ABNORMAL_STR = "笔记状态异常,请稍后查看"
|
||||
self.NOTE_ABNORMAL_STR = "Note status abnormal, please check later"
|
||||
self.NOTE_ABNORMAL_CODE = -510001
|
||||
self.playwright_page = playwright_page
|
||||
self.cookie_dict = cookie_dict
|
||||
self._extractor = XiaoHongShuExtractor()
|
||||
# 初始化代理池(来自 ProxyRefreshMixin)
|
||||
# Initialize proxy pool (from ProxyRefreshMixin)
|
||||
self.init_proxy_pool(proxy_ip_pool)
|
||||
|
||||
async def _pre_headers(self, url: str, params: Optional[Dict] = None, payload: Optional[Dict] = None) -> Dict:
|
||||
"""请求头参数签名(使用 playwright 注入方式)
|
||||
"""Request header parameter signing (using playwright injection method)
|
||||
|
||||
Args:
|
||||
url: 请求的URL
|
||||
params: GET请求的参数
|
||||
payload: POST请求的参数
|
||||
url: Request URL
|
||||
params: GET request parameters
|
||||
payload: POST request parameters
|
||||
|
||||
Returns:
|
||||
Dict: 请求头参数签名
|
||||
Dict: Signed request header parameters
|
||||
"""
|
||||
a1_value = self.cookie_dict.get("a1", "")
|
||||
|
||||
# 确定请求数据、方法和 URI
|
||||
# Determine request data, method and URI
|
||||
if params is not None:
|
||||
data = params
|
||||
method = "GET"
|
||||
@@ -91,7 +91,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
else:
|
||||
raise ValueError("params or payload is required")
|
||||
|
||||
# 使用 playwright 注入方式生成签名
|
||||
# Generate signature using playwright injection method
|
||||
signs = await sign_with_playwright(
|
||||
page=self.playwright_page,
|
||||
uri=url,
|
||||
@@ -112,16 +112,16 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
|
||||
async def request(self, method, url, **kwargs) -> Union[str, Any]:
|
||||
"""
|
||||
封装httpx的公共请求方法,对请求响应做一些处理
|
||||
Wrapper for httpx common request method, processes request response
|
||||
Args:
|
||||
method: 请求方法
|
||||
url: 请求的URL
|
||||
**kwargs: 其他请求参数,例如请求头、请求体等
|
||||
method: Request method
|
||||
url: Request URL
|
||||
**kwargs: Other request parameters, such as headers, body, etc.
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
# 每次请求前检测代理是否过期
|
||||
# Check if proxy is expired before each request
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
# return response.text
|
||||
@@ -133,7 +133,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
# someday someone maybe will bypass captcha
|
||||
verify_type = response.headers["Verifytype"]
|
||||
verify_uuid = response.headers["Verifyuuid"]
|
||||
msg = f"出现验证码,请求失败,Verifytype: {verify_type},Verifyuuid: {verify_uuid}, Response: {response}"
|
||||
msg = f"CAPTCHA appeared, request failed, Verifytype: {verify_type}, Verifyuuid: {verify_uuid}, Response: {response}"
|
||||
utils.logger.error(msg)
|
||||
raise Exception(msg)
|
||||
|
||||
@@ -150,10 +150,10 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
|
||||
async def get(self, uri: str, params: Optional[Dict] = None) -> Dict:
|
||||
"""
|
||||
GET请求,对请求头签名
|
||||
GET request, signs request headers
|
||||
Args:
|
||||
uri: 请求路由
|
||||
params: 请求参数
|
||||
uri: Request route
|
||||
params: Request parameters
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -167,10 +167,10 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
|
||||
async def post(self, uri: str, data: dict, **kwargs) -> Dict:
|
||||
"""
|
||||
POST请求,对请求头签名
|
||||
POST request, signs request headers
|
||||
Args:
|
||||
uri: 请求路由
|
||||
data: 请求体参数
|
||||
uri: Request route
|
||||
data: Request body parameters
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -186,7 +186,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
)
|
||||
|
||||
async def get_note_media(self, url: str) -> Union[bytes, None]:
|
||||
# 请求前检测代理是否过期
|
||||
# Check if proxy is expired before request
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
async with httpx.AsyncClient(proxy=self.proxy) as client:
|
||||
@@ -205,12 +205,12 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
) as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
|
||||
utils.logger.error(
|
||||
f"[XiaoHongShuClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}"
|
||||
) # 保留原始异常类型名称,以便开发者调试
|
||||
) # Keep original exception type name for developer debugging
|
||||
return None
|
||||
|
||||
async def pong(self) -> bool:
|
||||
"""
|
||||
用于检查登录态是否失效了
|
||||
Check if login state is still valid
|
||||
Returns:
|
||||
|
||||
"""
|
||||
@@ -218,7 +218,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
utils.logger.info("[XiaoHongShuClient.pong] Begin to pong xhs...")
|
||||
ping_flag = False
|
||||
try:
|
||||
note_card: Dict = await self.get_note_by_keyword(keyword="小红书")
|
||||
note_card: Dict = await self.get_note_by_keyword(keyword="Xiaohongshu")
|
||||
if note_card.get("items"):
|
||||
ping_flag = True
|
||||
except Exception as e:
|
||||
@@ -230,9 +230,9 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
|
||||
async def update_cookies(self, browser_context: BrowserContext):
|
||||
"""
|
||||
API客户端提供的更新cookies方法,一般情况下登录成功后会调用此方法
|
||||
Update cookies method provided by API client, usually called after successful login
|
||||
Args:
|
||||
browser_context: 浏览器上下文对象
|
||||
browser_context: Browser context object
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -251,13 +251,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
note_type: SearchNoteType = SearchNoteType.ALL,
|
||||
) -> Dict:
|
||||
"""
|
||||
根据关键词搜索笔记
|
||||
Search notes by keyword
|
||||
Args:
|
||||
keyword: 关键词参数
|
||||
page: 分页第几页
|
||||
page_size: 分页数据长度
|
||||
sort: 搜索结果排序指定
|
||||
note_type: 搜索的笔记类型
|
||||
keyword: Keyword parameter
|
||||
page: Page number
|
||||
page_size: Page data length
|
||||
sort: Search result sorting specification
|
||||
note_type: Type of note to search
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -280,11 +280,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
xsec_token: str,
|
||||
) -> Dict:
|
||||
"""
|
||||
获取笔记详情API
|
||||
Get note detail API
|
||||
Args:
|
||||
note_id:笔记ID
|
||||
xsec_source: 渠道来源
|
||||
xsec_token: 搜索关键字之后返回的比较列表中返回的token
|
||||
note_id: Note ID
|
||||
xsec_source: Channel source
|
||||
xsec_token: Token returned from search keyword result list
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -304,7 +304,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
if res and res.get("items"):
|
||||
res_dict: Dict = res["items"][0]["note_card"]
|
||||
return res_dict
|
||||
# 爬取频繁了可能会出现有的笔记能有结果有的没有
|
||||
# When crawling frequently, some notes may have results while others don't
|
||||
utils.logger.error(
|
||||
f"[XiaoHongShuClient.get_note_by_id] get note id:{note_id} empty and res:{res}"
|
||||
)
|
||||
@@ -317,11 +317,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
cursor: str = "",
|
||||
) -> Dict:
|
||||
"""
|
||||
获取一级评论的API
|
||||
Get first-level comments API
|
||||
Args:
|
||||
note_id: 笔记ID
|
||||
xsec_token: 验证token
|
||||
cursor: 分页游标
|
||||
note_id: Note ID
|
||||
xsec_token: Verification token
|
||||
cursor: Pagination cursor
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -345,13 +345,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
cursor: str = "",
|
||||
):
|
||||
"""
|
||||
获取指定父评论下的子评论的API
|
||||
Get sub-comments under specified parent comment API
|
||||
Args:
|
||||
note_id: 子评论的帖子ID
|
||||
root_comment_id: 根评论ID
|
||||
xsec_token: 验证token
|
||||
num: 分页数量
|
||||
cursor: 分页游标
|
||||
note_id: Post ID of sub-comments
|
||||
root_comment_id: Root comment ID
|
||||
xsec_token: Verification token
|
||||
num: Pagination quantity
|
||||
cursor: Pagination cursor
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -377,13 +377,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
max_count: int = 10,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
获取指定笔记下的所有一级评论,该方法会一直查找一个帖子下的所有评论信息
|
||||
Get all first-level comments under specified note, this method will continuously find all comment information under a post
|
||||
Args:
|
||||
note_id: 笔记ID
|
||||
xsec_token: 验证token
|
||||
crawl_interval: 爬取一次笔记的延迟单位(秒)
|
||||
callback: 一次笔记爬取结束后
|
||||
max_count: 一次笔记爬取的最大评论数量
|
||||
note_id: Note ID
|
||||
xsec_token: Verification token
|
||||
crawl_interval: Crawl delay per note (seconds)
|
||||
callback: Callback after one note crawl ends
|
||||
max_count: Maximum number of comments to crawl per note
|
||||
Returns:
|
||||
|
||||
"""
|
||||
@@ -425,12 +425,12 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
callback: Optional[Callable] = None,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
获取指定一级评论下的所有二级评论, 该方法会一直查找一级评论下的所有二级评论信息
|
||||
Get all second-level comments under specified first-level comments, this method will continuously find all second-level comment information under first-level comments
|
||||
Args:
|
||||
comments: 评论列表
|
||||
xsec_token: 验证token
|
||||
crawl_interval: 爬取一次评论的延迟单位(秒)
|
||||
callback: 一次评论爬取结束后
|
||||
comments: Comment list
|
||||
xsec_token: Verification token
|
||||
crawl_interval: Crawl delay per comment (seconds)
|
||||
callback: Callback after one comment crawl ends
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -487,18 +487,18 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
self, user_id: str, xsec_token: str = "", xsec_source: str = ""
|
||||
) -> Dict:
|
||||
"""
|
||||
通过解析网页版的用户主页HTML,获取用户个人简要信息
|
||||
PC端用户主页的网页存在window.__INITIAL_STATE__这个变量上的,解析它即可
|
||||
Get user profile brief information by parsing user homepage HTML
|
||||
The PC user homepage has window.__INITIAL_STATE__ variable, just parse it
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
xsec_token: 验证token (可选,如果URL中包含此参数则传入)
|
||||
xsec_source: 渠道来源 (可选,如果URL中包含此参数则传入)
|
||||
user_id: User ID
|
||||
xsec_token: Verification token (optional, pass if included in URL)
|
||||
xsec_source: Channel source (optional, pass if included in URL)
|
||||
|
||||
Returns:
|
||||
Dict: 创作者信息
|
||||
Dict: Creator information
|
||||
"""
|
||||
# 构建URI,如果有xsec参数则添加到URL中
|
||||
# Build URI, add xsec parameters to URL if available
|
||||
uri = f"/user/profile/{user_id}"
|
||||
if xsec_token and xsec_source:
|
||||
uri = f"{uri}?xsec_token={xsec_token}&xsec_source={xsec_source}"
|
||||
@@ -517,13 +517,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
xsec_source: str = "pc_feed",
|
||||
) -> Dict:
|
||||
"""
|
||||
获取博主的笔记
|
||||
Get creator's notes
|
||||
Args:
|
||||
creator: 博主ID
|
||||
cursor: 上一页最后一条笔记的ID
|
||||
page_size: 分页数据长度
|
||||
xsec_token: 验证token
|
||||
xsec_source: 渠道来源
|
||||
creator: Creator ID
|
||||
cursor: Last note ID from previous page
|
||||
page_size: Page data length
|
||||
xsec_token: Verification token
|
||||
xsec_source: Channel source
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -547,13 +547,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
xsec_source: str = "pc_feed",
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
获取指定用户下的所有发过的帖子,该方法会一直查找一个用户下的所有帖子信息
|
||||
Get all posts published by specified user, this method will continuously find all post information under a user
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
crawl_interval: 爬取一次的延迟单位(秒)
|
||||
callback: 一次分页爬取结束后的更新回调函数
|
||||
xsec_token: 验证token
|
||||
xsec_source: 渠道来源
|
||||
user_id: User ID
|
||||
crawl_interval: Crawl delay (seconds)
|
||||
callback: Update callback function after one pagination crawl ends
|
||||
xsec_token: Verification token
|
||||
xsec_source: Channel source
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -602,9 +602,9 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
|
||||
async def get_note_short_url(self, note_id: str) -> Dict:
|
||||
"""
|
||||
获取笔记的短链接
|
||||
Get note short URL
|
||||
Args:
|
||||
note_id: 笔记ID
|
||||
note_id: Note ID
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -622,7 +622,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
enable_cookie: bool = False,
|
||||
) -> Optional[Dict]:
|
||||
"""
|
||||
通过解析网页版的笔记详情页HTML,获取笔记详情, 该接口可能会出现失败的情况,这里尝试重试3次
|
||||
Get note details by parsing note detail page HTML, this interface may fail, retry 3 times here
|
||||
copy from https://github.com/ReaJason/xhs/blob/eb1c5a0213f6fbb592f0a2897ee552847c69ea2d/xhs/core.py#L217-L259
|
||||
thanks for ReaJason
|
||||
Args:
|
||||
|
||||
@@ -60,7 +60,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
# self.user_agent = utils.get_user_agent()
|
||||
self.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
||||
self.cdp_manager = None
|
||||
self.ip_proxy_pool = None # 代理IP池,用于代理自动刷新
|
||||
self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh
|
||||
|
||||
async def start(self) -> None:
|
||||
playwright_proxy_format, httpx_proxy_format = None, None
|
||||
@@ -70,9 +70,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
playwright_proxy_format, httpx_proxy_format = utils.format_proxy_info(ip_proxy_info)
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
# 根据配置选择启动模式
|
||||
# Choose launch mode based on configuration
|
||||
if config.ENABLE_CDP_MODE:
|
||||
utils.logger.info("[XiaoHongShuCrawler] 使用CDP模式启动浏览器")
|
||||
utils.logger.info("[XiaoHongShuCrawler] Launching browser using CDP mode")
|
||||
self.browser_context = await self.launch_browser_with_cdp(
|
||||
playwright,
|
||||
playwright_proxy_format,
|
||||
@@ -80,7 +80,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
headless=config.CDP_HEADLESS,
|
||||
)
|
||||
else:
|
||||
utils.logger.info("[XiaoHongShuCrawler] 使用标准模式启动浏览器")
|
||||
utils.logger.info("[XiaoHongShuCrawler] Launching browser using standard mode")
|
||||
# Launch a browser context.
|
||||
chromium = playwright.chromium
|
||||
self.browser_context = await self.launch_browser(
|
||||
@@ -95,7 +95,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
self.context_page = await self.browser_context.new_page()
|
||||
await self.context_page.goto(self.index_url)
|
||||
|
||||
# Create a client to interact with the xiaohongshu website.
|
||||
# Create a client to interact with the Xiaohongshu website.
|
||||
self.xhs_client = await self.create_xhs_client(httpx_proxy_format)
|
||||
if not await self.xhs_client.pong():
|
||||
login_obj = XiaoHongShuLogin(
|
||||
@@ -125,8 +125,8 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
|
||||
async def search(self) -> None:
|
||||
"""Search for notes and retrieve their comment information."""
|
||||
utils.logger.info("[XiaoHongShuCrawler.search] Begin search xiaohongshu keywords")
|
||||
xhs_limit_count = 20 # xhs limit page fixed value
|
||||
utils.logger.info("[XiaoHongShuCrawler.search] Begin search Xiaohongshu keywords")
|
||||
xhs_limit_count = 20 # Xiaohongshu limit page fixed value
|
||||
if config.CRAWLER_MAX_NOTES_COUNT < xhs_limit_count:
|
||||
config.CRAWLER_MAX_NOTES_COUNT = xhs_limit_count
|
||||
start_page = config.START_PAGE
|
||||
@@ -142,7 +142,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
continue
|
||||
|
||||
try:
|
||||
utils.logger.info(f"[XiaoHongShuCrawler.search] search xhs keyword: {keyword}, page: {page}")
|
||||
utils.logger.info(f"[XiaoHongShuCrawler.search] search Xiaohongshu keyword: {keyword}, page: {page}")
|
||||
note_ids: List[str] = []
|
||||
xsec_tokens: List[str] = []
|
||||
notes_res = await self.xhs_client.get_note_by_keyword(
|
||||
@@ -151,9 +151,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
page=page,
|
||||
sort=(SearchSortType(config.SORT_TYPE) if config.SORT_TYPE != "" else SearchSortType.GENERAL),
|
||||
)
|
||||
utils.logger.info(f"[XiaoHongShuCrawler.search] Search notes res:{notes_res}")
|
||||
utils.logger.info(f"[XiaoHongShuCrawler.search] Search notes response: {notes_res}")
|
||||
if not notes_res or not notes_res.get("has_more", False):
|
||||
utils.logger.info("No more content!")
|
||||
utils.logger.info("[XiaoHongShuCrawler.search] No more content!")
|
||||
break
|
||||
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
|
||||
task_list = [
|
||||
@@ -184,7 +184,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
|
||||
async def get_creators_and_notes(self) -> None:
|
||||
"""Get creator's notes and retrieve their comment information."""
|
||||
utils.logger.info("[XiaoHongShuCrawler.get_creators_and_notes] Begin get xiaohongshu creators")
|
||||
utils.logger.info("[XiaoHongShuCrawler.get_creators_and_notes] Begin get Xiaohongshu creators")
|
||||
for creator_url in config.XHS_CREATOR_ID_LIST:
|
||||
try:
|
||||
# Parse creator URL to get user_id and security tokens
|
||||
@@ -223,9 +223,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
await self.batch_get_note_comments(note_ids, xsec_tokens)
|
||||
|
||||
async def fetch_creator_notes_detail(self, note_list: List[Dict]):
|
||||
"""
|
||||
Concurrently obtain the specified post list and save the data
|
||||
"""
|
||||
"""Concurrently obtain the specified post list and save the data"""
|
||||
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
|
||||
task_list = [
|
||||
self.get_note_detail_async_task(
|
||||
@@ -243,11 +241,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
await self.get_notice_media(note_detail)
|
||||
|
||||
async def get_specified_notes(self):
|
||||
"""
|
||||
Get the information and comments of the specified post
|
||||
must be specified note_id, xsec_source, xsec_token⚠️⚠️⚠️
|
||||
Returns:
|
||||
"""Get the information and comments of the specified post
|
||||
|
||||
Note: Must specify note_id, xsec_source, xsec_token
|
||||
"""
|
||||
get_note_detail_task_list = []
|
||||
for full_note_url in config.XHS_SPECIFIED_NOTE_URL_LIST:
|
||||
@@ -356,8 +352,8 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
utils.logger.info(f"[XiaoHongShuCrawler.get_comments] Sleeping for {crawl_interval} seconds after fetching comments for note {note_id}")
|
||||
|
||||
async def create_xhs_client(self, httpx_proxy: Optional[str]) -> XiaoHongShuClient:
|
||||
"""Create xhs client"""
|
||||
utils.logger.info("[XiaoHongShuCrawler.create_xhs_client] Begin create xiaohongshu API client ...")
|
||||
"""Create Xiaohongshu client"""
|
||||
utils.logger.info("[XiaoHongShuCrawler.create_xhs_client] Begin create Xiaohongshu API client ...")
|
||||
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
|
||||
xhs_client_obj = XiaoHongShuClient(
|
||||
proxy=httpx_proxy,
|
||||
@@ -381,7 +377,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
},
|
||||
playwright_page=self.context_page,
|
||||
cookie_dict=cookie_dict,
|
||||
proxy_ip_pool=self.ip_proxy_pool, # 传递代理池用于自动刷新
|
||||
proxy_ip_pool=self.ip_proxy_pool, # Pass proxy pool for automatic refresh
|
||||
)
|
||||
return xhs_client_obj
|
||||
|
||||
@@ -422,9 +418,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
user_agent: Optional[str],
|
||||
headless: bool = True,
|
||||
) -> BrowserContext:
|
||||
"""
|
||||
使用CDP模式启动浏览器
|
||||
"""
|
||||
"""Launch browser using CDP mode"""
|
||||
try:
|
||||
self.cdp_manager = CDPBrowserManager()
|
||||
browser_context = await self.cdp_manager.launch_and_connect(
|
||||
@@ -434,21 +428,21 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
headless=headless,
|
||||
)
|
||||
|
||||
# 显示浏览器信息
|
||||
# Display browser information
|
||||
browser_info = await self.cdp_manager.get_browser_info()
|
||||
utils.logger.info(f"[XiaoHongShuCrawler] CDP浏览器信息: {browser_info}")
|
||||
utils.logger.info(f"[XiaoHongShuCrawler] CDP browser info: {browser_info}")
|
||||
|
||||
return browser_context
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[XiaoHongShuCrawler] CDP模式启动失败,回退到标准模式: {e}")
|
||||
# 回退到标准模式
|
||||
utils.logger.error(f"[XiaoHongShuCrawler] CDP mode launch failed, falling back to standard mode: {e}")
|
||||
# Fall back to standard mode
|
||||
chromium = playwright.chromium
|
||||
return await self.launch_browser(chromium, playwright_proxy, user_agent, headless)
|
||||
|
||||
async def close(self):
|
||||
"""Close browser context"""
|
||||
# 如果使用CDP模式,需要特殊处理
|
||||
# Special handling if using CDP mode
|
||||
if self.cdp_manager:
|
||||
await self.cdp_manager.cleanup()
|
||||
self.cdp_manager = None
|
||||
@@ -464,10 +458,10 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
await self.get_notice_video(note_detail)
|
||||
|
||||
async def get_note_images(self, note_item: Dict):
|
||||
"""
|
||||
get note images. please use get_notice_media
|
||||
:param note_item:
|
||||
:return:
|
||||
"""Get note images. Please use get_notice_media
|
||||
|
||||
Args:
|
||||
note_item: Note item dictionary
|
||||
"""
|
||||
if not config.ENABLE_GET_MEIDAS:
|
||||
return
|
||||
@@ -494,10 +488,10 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
await xhs_store.update_xhs_note_image(note_id, content, extension_file_name)
|
||||
|
||||
async def get_notice_video(self, note_item: Dict):
|
||||
"""
|
||||
get note videos. please use get_notice_media
|
||||
:param note_item:
|
||||
:return:
|
||||
"""Get note videos. Please use get_notice_media
|
||||
|
||||
Args:
|
||||
note_item: Note item dictionary
|
||||
"""
|
||||
if not config.ENABLE_GET_MEIDAS:
|
||||
return
|
||||
|
||||
@@ -29,16 +29,16 @@ class XiaoHongShuExtractor:
|
||||
pass
|
||||
|
||||
def extract_note_detail_from_html(self, note_id: str, html: str) -> Optional[Dict]:
|
||||
"""从html中提取笔记详情
|
||||
"""Extract note details from HTML
|
||||
|
||||
Args:
|
||||
html (str): html字符串
|
||||
html (str): HTML string
|
||||
|
||||
Returns:
|
||||
Dict: 笔记详情字典
|
||||
Dict: Note details dictionary
|
||||
"""
|
||||
if "noteDetailMap" not in html:
|
||||
# 这种情况要么是出了验证码了,要么是笔记不存在
|
||||
# Either a CAPTCHA appeared or the note doesn't exist
|
||||
return None
|
||||
|
||||
state = re.findall(r"window.__INITIAL_STATE__=({.*})</script>", html)[
|
||||
@@ -50,13 +50,13 @@ class XiaoHongShuExtractor:
|
||||
return None
|
||||
|
||||
def extract_creator_info_from_html(self, html: str) -> Optional[Dict]:
|
||||
"""从html中提取用户信息
|
||||
"""Extract user information from HTML
|
||||
|
||||
Args:
|
||||
html (str): html字符串
|
||||
html (str): HTML string
|
||||
|
||||
Returns:
|
||||
Dict: 用户信息字典
|
||||
Dict: User information dictionary
|
||||
"""
|
||||
match = re.search(
|
||||
r"<script>window.__INITIAL_STATE__=(.+)<\/script>", html, re.M
|
||||
|
||||
@@ -23,27 +23,27 @@ from typing import NamedTuple
|
||||
|
||||
|
||||
class FeedType(Enum):
|
||||
# 推荐
|
||||
# Recommend
|
||||
RECOMMEND = "homefeed_recommend"
|
||||
# 穿搭
|
||||
# Fashion
|
||||
FASION = "homefeed.fashion_v3"
|
||||
# 美食
|
||||
# Food
|
||||
FOOD = "homefeed.food_v3"
|
||||
# 彩妆
|
||||
# Cosmetics
|
||||
COSMETICS = "homefeed.cosmetics_v3"
|
||||
# 影视
|
||||
# Movie and TV
|
||||
MOVIE = "homefeed.movie_and_tv_v3"
|
||||
# 职场
|
||||
# Career
|
||||
CAREER = "homefeed.career_v3"
|
||||
# 情感
|
||||
# Emotion
|
||||
EMOTION = "homefeed.love_v3"
|
||||
# 家居
|
||||
# Home
|
||||
HOURSE = "homefeed.household_product_v3"
|
||||
# 游戏
|
||||
# Gaming
|
||||
GAME = "homefeed.gaming_v3"
|
||||
# 旅行
|
||||
# Travel
|
||||
TRAVEL = "homefeed.travel_v3"
|
||||
# 健身
|
||||
# Fitness
|
||||
FITNESS = "homefeed.fitness_v3"
|
||||
|
||||
|
||||
@@ -53,28 +53,27 @@ class NoteType(Enum):
|
||||
|
||||
|
||||
class SearchSortType(Enum):
|
||||
"""search sort type"""
|
||||
# default
|
||||
"""Search sort type"""
|
||||
# Default
|
||||
GENERAL = "general"
|
||||
# most popular
|
||||
# Most popular
|
||||
MOST_POPULAR = "popularity_descending"
|
||||
# Latest
|
||||
LATEST = "time_descending"
|
||||
|
||||
|
||||
class SearchNoteType(Enum):
|
||||
"""search note type
|
||||
"""
|
||||
# default
|
||||
"""Search note type"""
|
||||
# Default
|
||||
ALL = 0
|
||||
# only video
|
||||
# Only video
|
||||
VIDEO = 1
|
||||
# only image
|
||||
# Only image
|
||||
IMAGE = 2
|
||||
|
||||
|
||||
class Note(NamedTuple):
|
||||
"""note tuple"""
|
||||
"""Note tuple"""
|
||||
note_id: str
|
||||
title: str
|
||||
desc: str
|
||||
|
||||
@@ -297,13 +297,13 @@ def get_img_urls_by_trace_id(trace_id: str, format_type: str = "png"):
|
||||
|
||||
|
||||
def get_trace_id(img_url: str):
|
||||
# 浏览器端上传的图片多了 /spectrum/ 这个路径
|
||||
# Browser-uploaded images have an additional /spectrum/ path
|
||||
return f"spectrum/{img_url.split('/')[-1]}" if img_url.find("spectrum") != -1 else img_url.split("/")[-1]
|
||||
|
||||
|
||||
def parse_note_info_from_note_url(url: str) -> NoteUrlInfo:
|
||||
"""
|
||||
从小红书笔记url中解析出笔记信息
|
||||
Parse note information from Xiaohongshu note URL
|
||||
Args:
|
||||
url: "https://www.xiaohongshu.com/explore/66fad51c000000001b0224b8?xsec_token=AB3rO-QopW5sgrJ41GwN01WCXh6yWPxjSoFI9D5JIMgKw=&xsec_source=pc_search"
|
||||
Returns:
|
||||
@@ -318,44 +318,44 @@ def parse_note_info_from_note_url(url: str) -> NoteUrlInfo:
|
||||
|
||||
def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
|
||||
"""
|
||||
从小红书创作者主页URL中解析出创作者信息
|
||||
支持以下格式:
|
||||
1. 完整URL: "https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed"
|
||||
2. 纯ID: "5eb8e1d400000000010075ae"
|
||||
Parse creator information from Xiaohongshu creator homepage URL
|
||||
Supports the following formats:
|
||||
1. Full URL: "https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed"
|
||||
2. Pure ID: "5eb8e1d400000000010075ae"
|
||||
|
||||
Args:
|
||||
url: 创作者主页URL或user_id
|
||||
url: Creator homepage URL or user_id
|
||||
Returns:
|
||||
CreatorUrlInfo: 包含user_id, xsec_token, xsec_source的对象
|
||||
CreatorUrlInfo: Object containing user_id, xsec_token, xsec_source
|
||||
"""
|
||||
# 如果是纯ID格式(24位十六进制字符),直接返回
|
||||
# If it's a pure ID format (24 hexadecimal characters), return directly
|
||||
if len(url) == 24 and all(c in "0123456789abcdef" for c in url):
|
||||
return CreatorUrlInfo(user_id=url, xsec_token="", xsec_source="")
|
||||
|
||||
# 从URL中提取user_id: /user/profile/xxx
|
||||
# Extract user_id from URL: /user/profile/xxx
|
||||
import re
|
||||
user_pattern = r'/user/profile/([^/?]+)'
|
||||
match = re.search(user_pattern, url)
|
||||
if match:
|
||||
user_id = match.group(1)
|
||||
# 提取xsec_token和xsec_source参数
|
||||
# Extract xsec_token and xsec_source parameters
|
||||
params = extract_url_params_to_dict(url)
|
||||
xsec_token = params.get("xsec_token", "")
|
||||
xsec_source = params.get("xsec_source", "")
|
||||
return CreatorUrlInfo(user_id=user_id, xsec_token=xsec_token, xsec_source=xsec_source)
|
||||
|
||||
raise ValueError(f"无法从URL中解析出创作者信息: {url}")
|
||||
raise ValueError(f"Unable to parse creator info from URL: {url}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
_img_url = "https://sns-img-bd.xhscdn.com/7a3abfaf-90c1-a828-5de7-022c80b92aa3"
|
||||
# 获取一个图片地址在多个cdn下的url地址
|
||||
# Get image URL addresses under multiple CDNs for a single image
|
||||
# final_img_urls = get_img_urls_by_trace_id(get_trace_id(_img_url))
|
||||
final_img_url = get_img_url_by_trace_id(get_trace_id(_img_url))
|
||||
print(final_img_url)
|
||||
|
||||
# 测试创作者URL解析
|
||||
print("\n=== 创作者URL解析测试 ===")
|
||||
# Test creator URL parsing
|
||||
print("\n=== Creator URL Parsing Test ===")
|
||||
test_creator_urls = [
|
||||
"https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed",
|
||||
"5eb8e1d400000000010075ae",
|
||||
@@ -364,7 +364,7 @@ if __name__ == '__main__':
|
||||
try:
|
||||
result = parse_creator_info_from_url(url)
|
||||
print(f"✓ URL: {url[:80]}...")
|
||||
print(f" 结果: {result}\n")
|
||||
print(f" Result: {result}\n")
|
||||
except Exception as e:
|
||||
print(f"✗ URL: {url}")
|
||||
print(f" 错误: {e}\n")
|
||||
print(f" Error: {e}\n")
|
||||
|
||||
@@ -57,7 +57,7 @@ class XiaoHongShuLogin(AbstractLogin):
|
||||
"""
|
||||
|
||||
if "请通过验证" in await self.context_page.content():
|
||||
utils.logger.info("[XiaoHongShuLogin.check_login_state] 登录过程中出现验证码,请手动验证")
|
||||
utils.logger.info("[XiaoHongShuLogin.check_login_state] CAPTCHA appeared during login, please verify manually")
|
||||
|
||||
current_cookie = await self.browser_context.cookies()
|
||||
_, cookie_dict = utils.convert_cookies(current_cookie)
|
||||
@@ -83,14 +83,14 @@ class XiaoHongShuLogin(AbstractLogin):
|
||||
utils.logger.info("[XiaoHongShuLogin.login_by_mobile] Begin login xiaohongshu by mobile ...")
|
||||
await asyncio.sleep(1)
|
||||
try:
|
||||
# 小红书进入首页后,有可能不会自动弹出登录框,需要手动点击登录按钮
|
||||
# After entering Xiaohongshu homepage, the login dialog may not pop up automatically, need to manually click login button
|
||||
login_button_ele = await self.context_page.wait_for_selector(
|
||||
selector="xpath=//*[@id='app']/div[1]/div[2]/div[1]/ul/div[1]/button",
|
||||
timeout=5000
|
||||
)
|
||||
await login_button_ele.click()
|
||||
# 弹窗的登录对话框也有两种形态,一种是直接可以看到手机号和验证码的
|
||||
# 另一种是需要点击切换到手机登录的
|
||||
# The login dialog has two forms: one shows phone number and verification code directly
|
||||
# The other requires clicking to switch to phone login
|
||||
element = await self.context_page.wait_for_selector(
|
||||
selector='xpath=//div[@class="login-container"]//div[@class="other-method"]/div[1]',
|
||||
timeout=5000
|
||||
@@ -106,11 +106,11 @@ class XiaoHongShuLogin(AbstractLogin):
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
send_btn_ele = await login_container_ele.query_selector("label.auth-code > span")
|
||||
await send_btn_ele.click() # 点击发送验证码
|
||||
await send_btn_ele.click() # Click to send verification code
|
||||
sms_code_input_ele = await login_container_ele.query_selector("label.auth-code > input")
|
||||
submit_btn_ele = await login_container_ele.query_selector("div.input-container > button")
|
||||
cache_client = CacheFactory.create_cache(config.CACHE_TYPE_MEMORY)
|
||||
max_get_sms_code_time = 60 * 2 # 最长获取验证码的时间为2分钟
|
||||
max_get_sms_code_time = 60 * 2 # Maximum time to get verification code is 2 minutes
|
||||
no_logged_in_session = ""
|
||||
while max_get_sms_code_time > 0:
|
||||
utils.logger.info(f"[XiaoHongShuLogin.login_by_mobile] get sms code from redis remaining time {max_get_sms_code_time}s ...")
|
||||
@@ -125,15 +125,15 @@ class XiaoHongShuLogin(AbstractLogin):
|
||||
_, cookie_dict = utils.convert_cookies(current_cookie)
|
||||
no_logged_in_session = cookie_dict.get("web_session")
|
||||
|
||||
await sms_code_input_ele.fill(value=sms_code_value.decode()) # 输入短信验证码
|
||||
await sms_code_input_ele.fill(value=sms_code_value.decode()) # Enter SMS verification code
|
||||
await asyncio.sleep(0.5)
|
||||
agree_privacy_ele = self.context_page.locator("xpath=//div[@class='agreements']//*[local-name()='svg']")
|
||||
await agree_privacy_ele.click() # 点击同意隐私协议
|
||||
await agree_privacy_ele.click() # Click to agree to privacy policy
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await submit_btn_ele.click() # 点击登录
|
||||
await submit_btn_ele.click() # Click login
|
||||
|
||||
# todo ... 应该还需要检查验证码的正确性有可能输入的验证码不正确
|
||||
# TODO: Should also check if the verification code is correct, as it may be incorrect
|
||||
break
|
||||
|
||||
try:
|
||||
@@ -196,7 +196,7 @@ class XiaoHongShuLogin(AbstractLogin):
|
||||
"""login xiaohongshu website by cookies"""
|
||||
utils.logger.info("[XiaoHongShuLogin.login_by_cookies] Begin login xiaohongshu by cookie ...")
|
||||
for key, value in utils.convert_str_cookie_to_dict(self.cookie_str).items():
|
||||
if key != "web_session": # only set web_session cookie attr
|
||||
if key != "web_session": # Only set web_session cookie attribute
|
||||
continue
|
||||
await self.browser_context.add_cookies([{
|
||||
'name': key,
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
# 通过 Playwright 注入调用 window.mnsv2 生成小红书签名
|
||||
# Generate Xiaohongshu signature by calling window.mnsv2 via Playwright injection
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
@@ -30,18 +30,18 @@ from .xhs_sign import b64_encode, encode_utf8, get_trace_id, mrc
|
||||
|
||||
|
||||
def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method: str = "POST") -> str:
|
||||
"""构建待签名字符串
|
||||
|
||||
"""Build string to be signed
|
||||
|
||||
Args:
|
||||
uri: API路径
|
||||
data: 请求数据
|
||||
method: 请求方法 (GET 或 POST)
|
||||
|
||||
uri: API path
|
||||
data: Request data
|
||||
method: Request method (GET or POST)
|
||||
|
||||
Returns:
|
||||
待签名字符串
|
||||
String to be signed
|
||||
"""
|
||||
if method.upper() == "POST":
|
||||
# POST 请求使用 JSON 格式
|
||||
# POST request uses JSON format
|
||||
c = uri
|
||||
if data is not None:
|
||||
if isinstance(data, dict):
|
||||
@@ -50,10 +50,10 @@ def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method
|
||||
c += data
|
||||
return c
|
||||
else:
|
||||
# GET 请求使用查询字符串格式
|
||||
# GET request uses query string format
|
||||
if not data or (isinstance(data, dict) and len(data) == 0):
|
||||
return uri
|
||||
|
||||
|
||||
if isinstance(data, dict):
|
||||
params = []
|
||||
for key in data.keys():
|
||||
@@ -64,8 +64,8 @@ def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method
|
||||
value_str = str(value)
|
||||
else:
|
||||
value_str = ""
|
||||
# 使用URL编码(safe参数保留某些字符不编码)
|
||||
# 注意:httpx会对逗号、等号等字符进行编码,我们也需要同样处理
|
||||
# Use URL encoding (safe parameter preserves certain characters from encoding)
|
||||
# Note: httpx will encode commas, equals signs, etc., we need to handle the same way
|
||||
value_str = quote(value_str, safe='')
|
||||
params.append(f"{key}={value_str}")
|
||||
return f"{uri}?{'&'.join(params)}"
|
||||
@@ -75,12 +75,12 @@ def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method
|
||||
|
||||
|
||||
def _md5_hex(s: str) -> str:
|
||||
"""计算 MD5 哈希值"""
|
||||
"""Calculate MD5 hash value"""
|
||||
return hashlib.md5(s.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _build_xs_payload(x3_value: str, data_type: str = "object") -> str:
|
||||
"""构建 x-s 签名"""
|
||||
"""Build x-s signature"""
|
||||
s = {
|
||||
"x0": "4.2.1",
|
||||
"x1": "xhs-pc-web",
|
||||
@@ -92,7 +92,7 @@ def _build_xs_payload(x3_value: str, data_type: str = "object") -> str:
|
||||
|
||||
|
||||
def _build_xs_common(a1: str, b1: str, x_s: str, x_t: str) -> str:
|
||||
"""构建 x-s-common 请求头"""
|
||||
"""Build x-s-common request header"""
|
||||
payload = {
|
||||
"s0": 3,
|
||||
"s1": "",
|
||||
@@ -113,7 +113,7 @@ def _build_xs_common(a1: str, b1: str, x_s: str, x_t: str) -> str:
|
||||
|
||||
|
||||
async def get_b1_from_localstorage(page: Page) -> str:
|
||||
"""从 localStorage 获取 b1 值"""
|
||||
"""Get b1 value from localStorage"""
|
||||
try:
|
||||
local_storage = await page.evaluate("() => window.localStorage")
|
||||
return local_storage.get("b1", "")
|
||||
@@ -123,15 +123,15 @@ async def get_b1_from_localstorage(page: Page) -> str:
|
||||
|
||||
async def call_mnsv2(page: Page, sign_str: str, md5_str: str) -> str:
|
||||
"""
|
||||
通过 playwright 调用 window.mnsv2 函数
|
||||
Call window.mnsv2 function via playwright
|
||||
|
||||
Args:
|
||||
page: playwright Page 对象
|
||||
sign_str: 待签名字符串 (uri + JSON.stringify(data))
|
||||
md5_str: sign_str 的 MD5 哈希值
|
||||
page: playwright Page object
|
||||
sign_str: String to be signed (uri + JSON.stringify(data))
|
||||
md5_str: MD5 hash value of sign_str
|
||||
|
||||
Returns:
|
||||
mnsv2 返回的签名字符串
|
||||
Signature string returned by mnsv2
|
||||
"""
|
||||
sign_str_escaped = sign_str.replace("\\", "\\\\").replace("'", "\\'").replace("\n", "\\n")
|
||||
md5_str_escaped = md5_str.replace("\\", "\\\\").replace("'", "\\'")
|
||||
@@ -150,16 +150,16 @@ async def sign_xs_with_playwright(
|
||||
method: str = "POST",
|
||||
) -> str:
|
||||
"""
|
||||
通过 playwright 注入生成 x-s 签名
|
||||
Generate x-s signature via playwright injection
|
||||
|
||||
Args:
|
||||
page: playwright Page 对象(必须已打开小红书页面)
|
||||
uri: API 路径,如 "/api/sns/web/v1/search/notes"
|
||||
data: 请求数据(GET 的 params 或 POST 的 payload)
|
||||
method: 请求方法 (GET 或 POST)
|
||||
page: playwright Page object (must have Xiaohongshu page open)
|
||||
uri: API path, e.g., "/api/sns/web/v1/search/notes"
|
||||
data: Request data (GET params or POST payload)
|
||||
method: Request method (GET or POST)
|
||||
|
||||
Returns:
|
||||
x-s 签名字符串
|
||||
x-s signature string
|
||||
"""
|
||||
sign_str = _build_sign_string(uri, data, method)
|
||||
md5_str = _md5_hex(sign_str)
|
||||
@@ -176,17 +176,17 @@ async def sign_with_playwright(
|
||||
method: str = "POST",
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
通过 playwright 生成完整的签名请求头
|
||||
Generate complete signature request headers via playwright
|
||||
|
||||
Args:
|
||||
page: playwright Page 对象(必须已打开小红书页面)
|
||||
uri: API 路径
|
||||
data: 请求数据
|
||||
a1: cookie 中的 a1 值
|
||||
method: 请求方法 (GET 或 POST)
|
||||
page: playwright Page object (must have Xiaohongshu page open)
|
||||
uri: API path
|
||||
data: Request data
|
||||
a1: a1 value from cookie
|
||||
method: Request method (GET or POST)
|
||||
|
||||
Returns:
|
||||
包含 x-s, x-t, x-s-common, x-b3-traceid 的字典
|
||||
Dictionary containing x-s, x-t, x-s-common, x-b3-traceid
|
||||
"""
|
||||
b1 = await get_b1_from_localstorage(page)
|
||||
x_s = await sign_xs_with_playwright(page, uri, data, method)
|
||||
@@ -208,23 +208,23 @@ async def pre_headers_with_playwright(
|
||||
payload: Optional[Dict] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
使用 playwright 注入方式生成请求头签名
|
||||
可直接替换 client.py 中的 _pre_headers 方法
|
||||
Generate request header signature using playwright injection method
|
||||
Can directly replace _pre_headers method in client.py
|
||||
|
||||
Args:
|
||||
page: playwright Page 对象
|
||||
url: 请求 URL
|
||||
cookie_dict: cookie 字典
|
||||
params: GET 请求参数
|
||||
payload: POST 请求参数
|
||||
page: playwright Page object
|
||||
url: Request URL
|
||||
cookie_dict: Cookie dictionary
|
||||
params: GET request parameters
|
||||
payload: POST request parameters
|
||||
|
||||
Returns:
|
||||
签名后的请求头字典
|
||||
Signed request header dictionary
|
||||
"""
|
||||
a1_value = cookie_dict.get("a1", "")
|
||||
uri = urlparse(url).path
|
||||
|
||||
# 确定请求数据和方法
|
||||
# Determine request data and method
|
||||
if params is not None:
|
||||
data = params
|
||||
method = "GET"
|
||||
|
||||
@@ -16,19 +16,19 @@
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
# 小红书签名算法核心函数
|
||||
# 用于 playwright 注入方式生成签名
|
||||
# Xiaohongshu signature algorithm core functions
|
||||
# Used for generating signatures via playwright injection
|
||||
|
||||
import ctypes
|
||||
import random
|
||||
from urllib.parse import quote
|
||||
|
||||
# 自定义 Base64 字符表
|
||||
# 标准 Base64: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/
|
||||
# 小红书打乱顺序用于混淆
|
||||
# Custom Base64 character table
|
||||
# Standard Base64: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/
|
||||
# Xiaohongshu shuffled order for obfuscation
|
||||
BASE64_CHARS = list("ZmserbBoHQtNP+wOcza/LpngG8yJq42KWYj0DSfdikx3VT16IlUAFM97hECvuRX5")
|
||||
|
||||
# CRC32 查表
|
||||
# CRC32 lookup table
|
||||
CRC32_TABLE = [
|
||||
0, 1996959894, 3993919788, 2567524794, 124634137, 1886057615, 3915621685,
|
||||
2657392035, 249268274, 2044508324, 3772115230, 2547177864, 162941995,
|
||||
@@ -77,14 +77,14 @@ CRC32_TABLE = [
|
||||
|
||||
|
||||
def _right_shift_unsigned(num: int, bit: int = 0) -> int:
|
||||
"""JavaScript 无符号右移 (>>>) 的 Python 实现"""
|
||||
"""Python implementation of JavaScript unsigned right shift (>>>)"""
|
||||
val = ctypes.c_uint32(num).value >> bit
|
||||
MAX32INT = 4294967295
|
||||
return (val + (MAX32INT + 1)) % (2 * (MAX32INT + 1)) - MAX32INT - 1
|
||||
|
||||
|
||||
def mrc(e: str) -> int:
|
||||
"""CRC32 变体,用于 x-s-common 的 x9 字段"""
|
||||
"""CRC32 variant, used for x9 field in x-s-common"""
|
||||
o = -1
|
||||
for n in range(min(57, len(e))):
|
||||
o = CRC32_TABLE[(o & 255) ^ ord(e[n])] ^ _right_shift_unsigned(o, 8)
|
||||
@@ -92,7 +92,7 @@ def mrc(e: str) -> int:
|
||||
|
||||
|
||||
def _triplet_to_base64(e: int) -> str:
|
||||
"""将 24 位整数转换为 4 个 Base64 字符"""
|
||||
"""Convert 24-bit integer to 4 Base64 characters"""
|
||||
return (
|
||||
BASE64_CHARS[(e >> 18) & 63]
|
||||
+ BASE64_CHARS[(e >> 12) & 63]
|
||||
@@ -102,7 +102,7 @@ def _triplet_to_base64(e: int) -> str:
|
||||
|
||||
|
||||
def _encode_chunk(data: list, start: int, end: int) -> str:
|
||||
"""编码数据块"""
|
||||
"""Encode data chunk"""
|
||||
result = []
|
||||
for i in range(start, end, 3):
|
||||
c = ((data[i] << 16) & 0xFF0000) + ((data[i + 1] << 8) & 0xFF00) + (data[i + 2] & 0xFF)
|
||||
@@ -111,7 +111,7 @@ def _encode_chunk(data: list, start: int, end: int) -> str:
|
||||
|
||||
|
||||
def encode_utf8(s: str) -> list:
|
||||
"""将字符串编码为 UTF-8 字节列表"""
|
||||
"""Encode string to UTF-8 byte list"""
|
||||
encoded = quote(s, safe="~()*!.'")
|
||||
result = []
|
||||
i = 0
|
||||
@@ -126,7 +126,7 @@ def encode_utf8(s: str) -> list:
|
||||
|
||||
|
||||
def b64_encode(data: list) -> str:
|
||||
"""自定义 Base64 编码"""
|
||||
"""Custom Base64 encoding"""
|
||||
length = len(data)
|
||||
remainder = length % 3
|
||||
chunks = []
|
||||
@@ -148,5 +148,5 @@ def b64_encode(data: list) -> str:
|
||||
|
||||
|
||||
def get_trace_id() -> str:
|
||||
"""生成链路追踪 trace id"""
|
||||
"""Generate trace id for link tracing"""
|
||||
return "".join(random.choice("abcdef0123456789") for _ in range(16))
|
||||
|
||||
Reference in New Issue
Block a user