jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-03-05 13:40:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: httpx proxy format error

Browse Source 
feat: add a ip proxy provider
This commit is contained in:
程序员阿江(Relakkes)
2025-08-01 01:05:11 +08:00
parent 0024ce6ab4
commit 12450759d8
14 changed files with 1241 additions and 1070 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tools/crawler_util.py
View File

@@ -172,7 +172,7 @@ def match_interact_info_count(count_str: str) -> int:
return 0
def format_proxy_info(ip_proxy_info) -> Tuple[Optional[Dict], Optional[Dict]]:
def format_proxy_info(ip_proxy_info) -> Tuple[Optional[Dict], Optional[str]]:
"""format proxy info for playwright and httpx"""
# fix circular import issue
from proxy.proxy_ip_pool import IpInfoModel
@@ -183,9 +183,11 @@ def format_proxy_info(ip_proxy_info) -> Tuple[Optional[Dict], Optional[Dict]]:
"username": ip_proxy_info.user,
"password": ip_proxy_info.password,
}
httpx_proxy = {
f"{ip_proxy_info.protocol}": f"http://{ip_proxy_info.user}:{ip_proxy_info.password}@{ip_proxy_info.ip}:{ip_proxy_info.port}"
}
# httpx 0.28.1 需要直接传入代理URL字符串，而不是字典
if ip_proxy_info.user and ip_proxy_info.password:
httpx_proxy = f"http://{ip_proxy_info.user}:{ip_proxy_info.password}@{ip_proxy_info.ip}:{ip_proxy_info.port}"
else:
httpx_proxy = f"http://{ip_proxy_info.ip}:{ip_proxy_info.port}"
return playwright_proxy, httpx_proxy