jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-05-08 11:37:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

i18n: translate all Chinese comments, docstrings, and logger messages to English

Browse Source 
Comprehensive translation of Chinese text to English across the entire codebase:

- api/: FastAPI server documentation and logger messages
- cache/: Cache abstraction layer comments and docstrings
- database/: Database models and MongoDB store documentation
- media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu)
- model/: Data model documentation
- proxy/: Proxy pool and provider documentation
- store/: Data storage layer comments
- tools/: Utility functions and browser automation
- test/: Test file documentation

Preserved: Chinese disclaimer header (lines 10-18) for legal compliance

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
程序员阿江(Relakkes)
2025-12-26 23:27:19 +08:00
parent 1544d13dd5
commit 157ddfb21b
93 changed files with 1971 additions and 1955 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
proxy/__init__.py
View File

@@ -21,5 +21,5 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2023/12/2 14:37
# @Desc : IP代理池入口
# @Desc : IP proxy pool entry point
from .base_proxy import *

14
proxy/base_proxy.py
View File

@@ -21,8 +21,8 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2023/12/2 11:18
# @Desc : 爬虫 IP 获取实现
# @Url : 快代理HTTP实现，官方文档：https://www.kuaidaili.com/?ref=ldwkjqipvz6c
# @Desc : Crawler IP acquisition implementation
# @Url : KuaiDaili HTTP implementation, official documentation: https://www.kuaidaili.com/?ref=ldwkjqipvz6c
import json
from abc import ABC, abstractmethod
from typing import List
@@ -43,8 +43,8 @@ class ProxyProvider(ABC):
@abstractmethod
async def get_proxy(self, num: int) -> List[IpInfoModel]:
"""
获取 IP 的抽象方法，不同的 HTTP 代理商需要实现该方法
:param num: 提取的 IP 数量
Abstract method to get IP, different HTTP proxy providers need to implement this method
:param num: Number of IPs to extract
:return:
"""
raise NotImplementedError
@@ -57,7 +57,7 @@ class IpCache:
def set_ip(self, ip_key: str, ip_value_info: str, ex: int):
"""
设置IP并带有过期时间，到期之后由 redis 负责删除
Set IP with expiration time, Redis is responsible for deletion after expiration
:param ip_key:
:param ip_value_info:
:param ex:
@@ -67,8 +67,8 @@ class IpCache:
def load_all_ip(self, proxy_brand_name: str) -> List[IpInfoModel]:
"""
从 redis 中加载所有还未过期的 IP 信息
:param proxy_brand_name: 代理商名称
Load all unexpired IP information from Redis
:param proxy_brand_name: Proxy provider name
:return:
"""
all_ip_list: List[IpInfoModel] = []

30
proxy/providers/jishu_http_proxy.py
View File

@@ -20,7 +20,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/4/5 09:32
# @Desc : 已废弃！！！！！倒闭了！！！极速HTTP 代理IP实现. 请使用快代理实现（proxy/providers/kuaidl_proxy.py）
# @Desc : Deprecated!!!!! Shut down!!! JiSu HTTP proxy IP implementation. Please use KuaiDaili implementation (proxy/providers/kuaidl_proxy.py)
import os
from typing import Dict, List
from urllib.parse import urlencode
@@ -36,20 +36,20 @@ class JiSuHttpProxy(ProxyProvider):
def __init__(self, key: str, crypto: str, time_validity_period: int):
"""
极速HTTP 代理IP实现
:param key: 提取key值 (去官网注册后获取)
:param crypto: 加密签名 (去官网注册后获取)
JiSu HTTP proxy IP implementation
:param key: Extraction key value (obtain after registering on the official website)
:param crypto: Encryption signature (obtain after registering on the official website)
"""
self.proxy_brand_name = "JISUHTTP"
self.api_path = "https://api.jisuhttp.com"
self.params = {
"key": key,
"crypto": crypto,
"time": time_validity_period, # IP使用时长，支持3、5、10、15、30分钟时效
"type": "json", # 数据结果为json
"port": "2", # IP协议：1:HTTP、2:HTTPS、3:SOCKS5
"pw": "1", # 是否使用账密验证， 1：是，0：否，否表示白名单验证；默认为0
"se": "1", # 返回JSON格式时是否显示IP过期时间， 1：显示，0：不显示；默认为0
"time": time_validity_period, # IP usage duration, supports 3, 5, 10, 15, 30 minute validity
"type": "json", # Data result is json
"port": "2", # IP protocol: 1:HTTP, 2:HTTPS, 3:SOCKS5
"pw": "1", # Whether to use account password authentication, 1: yes, 0: no, no means whitelist authentication; default is 0
"se": "1", # Whether to show IP expiration time when returning JSON format, 1: show, 0: don't show; default is 0
}
self.ip_cache = IpCache()
@@ -59,12 +59,12 @@ class JiSuHttpProxy(ProxyProvider):
:return:
"""
# 优先从缓存中拿 IP
# Prioritize getting IP from cache
ip_cache_list = self.ip_cache.load_all_ip(proxy_brand_name=self.proxy_brand_name)
if len(ip_cache_list) >= num:
return ip_cache_list[:num]
# 如果缓存中的数量不够，从IP代理商获取补上，再存入缓存中
# If the quantity in cache is insufficient, get from IP provider to supplement, then store in cache
need_get_count = num - len(ip_cache_list)
self.params.update({"num": need_get_count})
ip_infos = []
@@ -97,12 +97,12 @@ class JiSuHttpProxy(ProxyProvider):
def new_jisu_http_proxy() -> JiSuHttpProxy:
"""
构造极速HTTP实例
Construct JiSu HTTP instance
Returns:
"""
return JiSuHttpProxy(
key=os.getenv("jisu_key", ""), # 通过环境变量的方式获取极速HTTPIP提取key值
crypto=os.getenv("jisu_crypto", ""), # 通过环境变量的方式获取极速HTTPIP提取加密签名
time_validity_period=30 # 30分钟（最长时效）
key=os.getenv("jisu_key", ""), # Get JiSu HTTP IP extraction key value through environment variable
crypto=os.getenv("jisu_crypto", ""), # Get JiSu HTTP IP extraction encryption signature through environment variable
time_validity_period=30 # 30 minutes (maximum validity)
)

42
proxy/providers/kuaidl_proxy.py
View File

@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/4/5 09:43
# @Desc : 快代理HTTP实现，官方文档：https://www.kuaidaili.com/?ref=ldwkjqipvz6c
# @Desc : KuaiDaili HTTP implementation, official documentation: https://www.kuaidaili.com/?ref=ldwkjqipvz6c
import os
import re
from typing import Dict, List
@@ -33,19 +33,19 @@ from proxy import IpCache, IpInfoModel, ProxyProvider
from proxy.types import ProviderNameEnum
from tools import utils
# 快代理的IP代理过期时间向前推移5秒，避免临界时间使用失败
# KuaiDaili IP proxy expiration time is moved forward by 5 seconds to avoid critical time usage failure
DELTA_EXPIRED_SECOND = 5
class KuaidailiProxyModel(BaseModel):
ip: str = Field("ip")
port: int = Field("端口")
expire_ts: int = Field("过期时间，单位秒，多少秒后过期")
port: int = Field("port")
expire_ts: int = Field("Expiration time, in seconds, how many seconds until expiration")
def parse_kuaidaili_proxy(proxy_info: str) -> KuaidailiProxyModel:
"""
解析快代理的IP信息
Parse KuaiDaili IP information
Args:
proxy_info:
@@ -94,7 +94,7 @@ class KuaiDaiLiProxy(ProxyProvider):
async def get_proxy(self, num: int) -> List[IpInfoModel]:
"""
快代理实现
KuaiDaili implementation
Args:
num:
@@ -103,12 +103,12 @@ class KuaiDaiLiProxy(ProxyProvider):
"""
uri = "/api/getdps/"
# 优先从缓存中拿 IP
# Prioritize getting IP from cache
ip_cache_list = self.ip_cache.load_all_ip(proxy_brand_name=self.proxy_brand_name)
if len(ip_cache_list) >= num:
return ip_cache_list[:num]
# 如果缓存中的数量不够，从IP代理商获取补上，再存入缓存中
# If the quantity in cache is insufficient, get from IP provider to supplement, then store in cache
need_get_count = num - len(ip_cache_list)
self.params.update({"num": need_get_count})
@@ -128,8 +128,8 @@ class KuaiDaiLiProxy(ProxyProvider):
proxy_list: List[str] = ip_response.get("data", {}).get("proxy_list")
for proxy in proxy_list:
proxy_model = parse_kuaidaili_proxy(proxy)
# expire_ts是相对时间（秒数），需要转换为绝对时间戳
# 提前DELTA_EXPIRED_SECOND秒认为过期，避免临界时间使用失败
# expire_ts is relative time (seconds), needs to be converted to absolute timestamp
# Consider expired DELTA_EXPIRED_SECOND seconds in advance to avoid critical time usage failure
ip_info_model = IpInfoModel(
ip=proxy_model.ip,
port=proxy_model.port,
@@ -139,7 +139,7 @@ class KuaiDaiLiProxy(ProxyProvider):
)
ip_key = f"{self.proxy_brand_name}_{ip_info_model.ip}_{ip_info_model.port}"
# 缓存过期时间使用相对时间（秒数），也需要减去缓冲时间
# Cache expiration time uses relative time (seconds), also needs to subtract buffer time
self.ip_cache.set_ip(ip_key, ip_info_model.model_dump_json(), ex=proxy_model.expire_ts - DELTA_EXPIRED_SECOND)
ip_infos.append(ip_info_model)
@@ -148,19 +148,19 @@ class KuaiDaiLiProxy(ProxyProvider):
def new_kuai_daili_proxy() -> KuaiDaiLiProxy:
"""
构造快代理HTTP实例
支持两种环境变量命名格式：
1. 大写格式：KDL_SECERT_ID, KDL_SIGNATURE, KDL_USER_NAME, KDL_USER_PWD
2. 小写格式：kdl_secret_id, kdl_signature, kdl_user_name, kdl_user_pwd
优先使用大写格式，如果不存在则使用小写格式
Construct KuaiDaili HTTP instance
Supports two environment variable naming formats:
1. Uppercase format: KDL_SECERT_ID, KDL_SIGNATURE, KDL_USER_NAME, KDL_USER_PWD
2. Lowercase format: kdl_secret_id, kdl_signature, kdl_user_name, kdl_user_pwd
Prioritize uppercase format, use lowercase format if not exists
Returns:
"""
# 支持大小写两种环境变量格式，优先使用大写
kdl_secret_id = os.getenv("KDL_SECERT_ID") or os.getenv("kdl_secret_id", "你的快代理secert_id")
kdl_signature = os.getenv("KDL_SIGNATURE") or os.getenv("kdl_signature", "你的快代理签名")
kdl_user_name = os.getenv("KDL_USER_NAME") or os.getenv("kdl_user_name", "你的快代理用户名")
kdl_user_pwd = os.getenv("KDL_USER_PWD") or os.getenv("kdl_user_pwd", "你的快代理密码")
# Support both uppercase and lowercase environment variable formats, prioritize uppercase
kdl_secret_id = os.getenv("KDL_SECERT_ID") or os.getenv("kdl_secret_id", "your_kuaidaili_secret_id")
kdl_signature = os.getenv("KDL_SIGNATURE") or os.getenv("kdl_signature", "your_kuaidaili_signature")
kdl_user_name = os.getenv("KDL_USER_NAME") or os.getenv("kdl_user_name", "your_kuaidaili_username")
kdl_user_pwd = os.getenv("KDL_USER_PWD") or os.getenv("kdl_user_pwd", "your_kuaidaili_password")
return KuaiDaiLiProxy(
kdl_secret_id=kdl_secret_id,

36
proxy/providers/wandou_http_proxy.py
View File

@@ -20,7 +20,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2025/7/31
# @Desc : 豌豆HTTP 代理IP实现
# @Desc : WanDou HTTP proxy IP implementation
import os
from typing import Dict, List
from urllib.parse import urlencode
@@ -36,9 +36,9 @@ class WanDouHttpProxy(ProxyProvider):
def __init__(self, app_key: str, num: int = 100):
"""
豌豆HTTP 代理IP实现
:param app_key: 开放的app_key,可以通过用户中心获取
:param num: 单次提取IP数量,最大100
WanDou HTTP proxy IP implementation
:param app_key: Open app_key, can be obtained through user center
:param num: Number of IPs extracted at once, maximum 100
"""
self.proxy_brand_name = "WANDOUHTTP"
self.api_path = "https://api.wandouapp.com/"
@@ -54,16 +54,16 @@ class WanDouHttpProxy(ProxyProvider):
:return:
"""
# 优先从缓存中拿 IP
# Prioritize getting IP from cache
ip_cache_list = self.ip_cache.load_all_ip(
proxy_brand_name=self.proxy_brand_name
)
if len(ip_cache_list) >= num:
return ip_cache_list[:num]
# 如果缓存中的数量不够，从IP代理商获取补上，再存入缓存中
# If the quantity in cache is insufficient, get from IP provider to supplement, then store in cache
need_get_count = num - len(ip_cache_list)
self.params.update({"num": min(need_get_count, 100)}) # 最大100
self.params.update({"num": min(need_get_count, 100)}) # Maximum 100
ip_infos = []
async with httpx.AsyncClient() as client:
url = self.api_path + "?" + urlencode(self.params)
@@ -82,7 +82,7 @@ class WanDouHttpProxy(ProxyProvider):
ip_info_model = IpInfoModel(
ip=ip_item.get("ip"),
port=ip_item.get("port"),
user="", # 豌豆HTTP不需要用户名密码认证
user="", # WanDou HTTP does not require username password authentication
password="",
expired_time_ts=utils.get_unix_time_from_time_str(
ip_item.get("expire_time")
@@ -96,27 +96,27 @@ class WanDouHttpProxy(ProxyProvider):
)
else:
error_msg = res_dict.get("msg", "unknown error")
# 处理具体错误码
# Handle specific error codes
error_code = res_dict.get("code")
if error_code == 10001:
error_msg = "通用错误，具体错误信息查看msg内容"
error_msg = "General error, check msg content for specific error information"
elif error_code == 10048:
error_msg = "没有可用套餐"
error_msg = "No available package"
raise IpGetError(f"{error_msg} (code: {error_code})")
return ip_cache_list + ip_infos
def new_wandou_http_proxy() -> WanDouHttpProxy:
"""
构造豌豆HTTP实例
支持两种环境变量命名格式：
1. 大写格式：WANDOU_APP_KEY
2. 小写格式：wandou_app_key
优先使用大写格式，如果不存在则使用小写格式
Construct WanDou HTTP instance
Supports two environment variable naming formats:
1. Uppercase format: WANDOU_APP_KEY
2. Lowercase format: wandou_app_key
Prioritize uppercase format, use lowercase format if not exists
Returns:
"""
# 支持大小写两种环境变量格式，优先使用大写
app_key = os.getenv("WANDOU_APP_KEY") or os.getenv("wandou_app_key", "你的豌豆HTTP app_key")
# Support both uppercase and lowercase environment variable formats, prioritize uppercase
app_key = os.getenv("WANDOU_APP_KEY") or os.getenv("wandou_app_key", "your_wandou_http_app_key")
return WanDouHttpProxy(app_key=app_key)

40
proxy/proxy_ip_pool.py
View File

@@ -20,7 +20,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2023/12/2 13:45
# @Desc : ip代理池实现
# @Desc : IP proxy pool implementation
import random
from typing import Dict, List
@@ -50,16 +50,16 @@ class ProxyIpPool:
enable_validate_ip:
ip_provider:
"""
self.valid_ip_url = "https://echo.apifox.cn/" # 验证 IP 是否有效的地址
self.valid_ip_url = "https://echo.apifox.cn/" # URL to validate if IP is valid
self.ip_pool_count = ip_pool_count
self.enable_validate_ip = enable_validate_ip
self.proxy_list: List[IpInfoModel] = []
self.ip_provider: ProxyProvider = ip_provider
self.current_proxy: IpInfoModel | None = None # 当前正在使用的代理
self.current_proxy: IpInfoModel | None = None # Currently used proxy
async def load_proxies(self) -> None:
"""
加载IP代理
Load IP proxies
Returns:
"""
@@ -67,7 +67,7 @@ class ProxyIpPool:
async def _is_valid_proxy(self, proxy: IpInfoModel) -> bool:
"""
验证代理IP是否有效
Validate if proxy IP is valid
:param proxy:
:return:
"""
@@ -75,7 +75,7 @@ class ProxyIpPool:
f"[ProxyIpPool._is_valid_proxy] testing {proxy.ip} is it valid "
)
try:
# httpx 0.28.1 需要直接传入代理URL字符串，而不是字典
# httpx 0.28.1 requires passing proxy URL string directly, not a dictionary
if proxy.user and proxy.password:
proxy_url = f"http://{proxy.user}:{proxy.password}@{proxy.ip}:{proxy.port}"
else:
@@ -96,29 +96,29 @@ class ProxyIpPool:
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
async def get_proxy(self) -> IpInfoModel:
"""
从代理池中随机提取一个代理IP
Randomly extract a proxy IP from the proxy pool
:return:
"""
if len(self.proxy_list) == 0:
await self._reload_proxies()
proxy = random.choice(self.proxy_list)
self.proxy_list.remove(proxy) # 取出来一个IP就应该移出掉
self.proxy_list.remove(proxy) # Remove an IP once extracted
if self.enable_validate_ip:
if not await self._is_valid_proxy(proxy):
raise Exception(
"[ProxyIpPool.get_proxy] current ip invalid and again get it"
)
self.current_proxy = proxy # 保存当前使用的代理
self.current_proxy = proxy # Save currently used proxy
return proxy
def is_current_proxy_expired(self, buffer_seconds: int = 30) -> bool:
"""
检测当前代理是否已过期
Check if current proxy has expired
Args:
buffer_seconds: 缓冲时间（秒），提前多少秒认为已过期
buffer_seconds: Buffer time (seconds), how many seconds ahead to consider expired
Returns:
bool: True表示已过期或没有当前代理，False表示仍然有效
bool: True means expired or no current proxy, False means still valid
"""
if self.current_proxy is None:
return True
@@ -126,12 +126,12 @@ class ProxyIpPool:
async def get_or_refresh_proxy(self, buffer_seconds: int = 30) -> IpInfoModel:
"""
获取当前代理，如果已过期则自动刷新
每次发起请求前调用此方法来确保代理有效
Get current proxy, automatically refresh if expired
Call this method before each request to ensure proxy is valid
Args:
buffer_seconds: 缓冲时间（秒），提前多少秒认为已过期
buffer_seconds: Buffer time (seconds), how many seconds ahead to consider expired
Returns:
IpInfoModel: 有效的代理IP信息
IpInfoModel: Valid proxy IP information
"""
if self.is_current_proxy_expired(buffer_seconds):
utils.logger.info(
@@ -142,7 +142,7 @@ class ProxyIpPool:
async def _reload_proxies(self):
"""
# 重新加载代理池
Reload proxy pool
:return:
"""
self.proxy_list = []
@@ -157,9 +157,9 @@ IpProxyProvider: Dict[str, ProxyProvider] = {
async def create_ip_pool(ip_pool_count: int, enable_validate_ip: bool) -> ProxyIpPool:
"""
创建 IP 代理池
:param ip_pool_count: ip池子的数量
:param enable_validate_ip: 是否开启验证IP代理
Create IP proxy pool
:param ip_pool_count: Number of IPs in the pool
:param enable_validate_ip: Whether to enable IP proxy validation
:return:
"""
pool = ProxyIpPool(

26
proxy/proxy_mixin.py
View File

@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2025/11/25
# @Desc : 代理自动刷新 Mixin 类，供各平台 client 使用
# @Desc : Auto-refresh proxy Mixin class for use by various platform clients
from typing import TYPE_CHECKING, Optional
@@ -33,31 +33,31 @@ if TYPE_CHECKING:
class ProxyRefreshMixin:
"""
代理自动刷新 Mixin 类
Auto-refresh proxy Mixin class
使用方法：
1. 让 client 类继承此 Mixin
2. 在 client 的 __init__ 中调用 init_proxy_pool(proxy_ip_pool)
3. 在每次 request 方法调用前调用 await _refresh_proxy_if_expired()
Usage:
1. Let client class inherit this Mixin
2. Call init_proxy_pool(proxy_ip_pool) in client's __init__
3. Call await _refresh_proxy_if_expired() before each request method call
要求：
- client 类必须有 self.proxy 属性来存储当前代理URL
Requirements:
- client class must have self.proxy attribute to store current proxy URL
"""
_proxy_ip_pool: Optional["ProxyIpPool"] = None
def init_proxy_pool(self, proxy_ip_pool: Optional["ProxyIpPool"]) -> None:
"""
初始化代理池引用
Initialize proxy pool reference
Args:
proxy_ip_pool: 代理IP池实例
proxy_ip_pool: Proxy IP pool instance
"""
self._proxy_ip_pool = proxy_ip_pool
async def _refresh_proxy_if_expired(self) -> None:
"""
检测代理是否过期，如果过期则自动刷新
每次发起请求前调用此方法来确保代理有效
Check if proxy has expired, automatically refresh if so
Call this method before each request to ensure proxy is valid
"""
if self._proxy_ip_pool is None:
return
@@ -67,7 +67,7 @@ class ProxyRefreshMixin:
f"[{self.__class__.__name__}._refresh_proxy_if_expired] Proxy expired, refreshing..."
)
new_proxy = await self._proxy_ip_pool.get_or_refresh_proxy()
# 更新 httpx 代理URL
# Update httpx proxy URL
if new_proxy.user and new_proxy.password:
self.proxy = f"http://{new_proxy.user}:{new_proxy.password}@{new_proxy.ip}:{new_proxy.port}"
else:

18
proxy/types.py
View File

@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2024/4/5 10:18
# @Desc : 基础类型
# @Desc : Basic types
import time
from enum import Enum
from typing import Optional
@@ -38,19 +38,19 @@ class IpInfoModel(BaseModel):
"""Unified IP model"""
ip: str = Field(title="ip")
port: int = Field(title="端口")
user: str = Field(title="IP代理认证的用户名")
protocol: str = Field(default="https://", title="代理IP的协议")
password: str = Field(title="IP代理认证用户的密码")
expired_time_ts: Optional[int] = Field(default=None, title="IP 过期时间")
port: int = Field(title="port")
user: str = Field(title="Username for IP proxy authentication")
protocol: str = Field(default="https://", title="Protocol for proxy IP")
password: str = Field(title="Password for IP proxy authentication user")
expired_time_ts: Optional[int] = Field(default=None, title="IP expiration time")
def is_expired(self, buffer_seconds: int = 30) -> bool:
"""
检测代理IP是否已过期
Check if proxy IP has expired
Args:
buffer_seconds: 缓冲时间（秒），提前多少秒认为已过期，避免临界时间请求失败
buffer_seconds: Buffer time (seconds), how many seconds ahead to consider expired to avoid critical time request failures
Returns:
bool: True表示已过期或即将过期，False表示仍然有效
bool: True means expired or about to expire, False means still valid
"""
if self.expired_time_ts is None:
return False