mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-03-19 09:47:30 +08:00
fix: extend make_async_client to proxy provider and IP pool
Migrate remaining httpx.AsyncClient call sites in proxy/ package to use make_async_client(), completing the DISABLE_SSL_VERIFY coverage across all outbound HTTP requests in the project. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -30,6 +30,7 @@ import httpx
|
||||
from proxy import IpCache, IpGetError, ProxyProvider
|
||||
from proxy.types import IpInfoModel
|
||||
from tools import utils
|
||||
from tools.httpx_util import make_async_client
|
||||
|
||||
|
||||
class JiSuHttpProxy(ProxyProvider):
|
||||
@@ -68,7 +69,7 @@ class JiSuHttpProxy(ProxyProvider):
|
||||
need_get_count = num - len(ip_cache_list)
|
||||
self.params.update({"num": need_get_count})
|
||||
ip_infos = []
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with make_async_client() as client:
|
||||
url = self.api_path + "/fetchips" + '?' + urlencode(self.params)
|
||||
utils.logger.info(f"[JiSuHttpProxy.get_proxy] get ip proxy url:{url}")
|
||||
response = await client.get(url, headers={
|
||||
|
||||
@@ -28,6 +28,7 @@ from typing import Dict, List
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field
|
||||
from tools.httpx_util import make_async_client
|
||||
|
||||
from proxy import IpCache, IpInfoModel, ProxyProvider
|
||||
from proxy.types import ProviderNameEnum
|
||||
@@ -113,7 +114,7 @@ class KuaiDaiLiProxy(ProxyProvider):
|
||||
self.params.update({"num": need_get_count})
|
||||
|
||||
ip_infos: List[IpInfoModel] = []
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with make_async_client() as client:
|
||||
response = await client.get(self.api_base + uri, params=self.params)
|
||||
|
||||
if response.status_code != 200:
|
||||
|
||||
@@ -30,6 +30,7 @@ import httpx
|
||||
from proxy import IpCache, IpGetError, ProxyProvider
|
||||
from proxy.types import IpInfoModel
|
||||
from tools import utils
|
||||
from tools.httpx_util import make_async_client
|
||||
|
||||
|
||||
class WanDouHttpProxy(ProxyProvider):
|
||||
@@ -65,7 +66,7 @@ class WanDouHttpProxy(ProxyProvider):
|
||||
need_get_count = num - len(ip_cache_list)
|
||||
self.params.update({"num": min(need_get_count, 100)}) # Maximum 100
|
||||
ip_infos = []
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with make_async_client() as client:
|
||||
url = self.api_path + "?" + urlencode(self.params)
|
||||
utils.logger.info(f"[WanDouHttpProxy.get_proxy] get ip proxy url:{url}")
|
||||
response = await client.get(
|
||||
|
||||
@@ -26,6 +26,7 @@ from typing import Dict, List
|
||||
|
||||
import httpx
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||
from tools.httpx_util import make_async_client
|
||||
|
||||
import config
|
||||
from proxy.providers import (
|
||||
@@ -81,7 +82,7 @@ class ProxyIpPool:
|
||||
else:
|
||||
proxy_url = f"http://{proxy.ip}:{proxy.port}"
|
||||
|
||||
async with httpx.AsyncClient(proxy=proxy_url) as client:
|
||||
async with make_async_client(proxy=proxy_url) as client:
|
||||
response = await client.get(self.valid_ip_url)
|
||||
if response.status_code == 200:
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user