jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-03-19 17:57:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: extend make_async_client to proxy provider and IP pool

Browse Source 
Migrate remaining httpx.AsyncClient call sites in proxy/ package to
use make_async_client(), completing the DISABLE_SSL_VERIFY coverage
across all outbound HTTP requests in the project.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Wei Liu
2026-03-18 12:39:35 +13:00
parent 125e02a4b9
commit dd327f068e
4 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
proxy/providers/jishu_http_proxy.py
View File

@@ -30,6 +30,7 @@ import httpx
from proxy import IpCache, IpGetError, ProxyProvider from proxy import IpCache, IpGetError, ProxyProvider
from proxy.types import IpInfoModel from proxy.types import IpInfoModel
from tools import utils from tools import utils
from tools.httpx_util import make_async_client
class JiSuHttpProxy(ProxyProvider): class JiSuHttpProxy(ProxyProvider):
@@ -68,7 +69,7 @@ class JiSuHttpProxy(ProxyProvider):
need_get_count = num - len(ip_cache_list) need_get_count = num - len(ip_cache_list)
self.params.update({"num": need_get_count}) self.params.update({"num": need_get_count})
ip_infos = [] ip_infos = []
async with httpx.AsyncClient() as client: async with make_async_client() as client:
url = self.api_path + "/fetchips" + '?' + urlencode(self.params) url = self.api_path + "/fetchips" + '?' + urlencode(self.params)
utils.logger.info(f"[JiSuHttpProxy.get_proxy] get ip proxy url:{url}") utils.logger.info(f"[JiSuHttpProxy.get_proxy] get ip proxy url:{url}")
response = await client.get(url, headers={ response = await client.get(url, headers={

3
proxy/providers/kuaidl_proxy.py
View File

@@ -28,6 +28,7 @@ from typing import Dict, List
import httpx import httpx
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from tools.httpx_util import make_async_client
from proxy import IpCache, IpInfoModel, ProxyProvider from proxy import IpCache, IpInfoModel, ProxyProvider
from proxy.types import ProviderNameEnum from proxy.types import ProviderNameEnum
@@ -113,7 +114,7 @@ class KuaiDaiLiProxy(ProxyProvider):
self.params.update({"num": need_get_count}) self.params.update({"num": need_get_count})
ip_infos: List[IpInfoModel] = [] ip_infos: List[IpInfoModel] = []
async with httpx.AsyncClient() as client: async with make_async_client() as client:
response = await client.get(self.api_base + uri, params=self.params) response = await client.get(self.api_base + uri, params=self.params)
if response.status_code != 200: if response.status_code != 200:

3
proxy/providers/wandou_http_proxy.py
View File

@@ -30,6 +30,7 @@ import httpx
from proxy import IpCache, IpGetError, ProxyProvider from proxy import IpCache, IpGetError, ProxyProvider
from proxy.types import IpInfoModel from proxy.types import IpInfoModel
from tools import utils from tools import utils
from tools.httpx_util import make_async_client
class WanDouHttpProxy(ProxyProvider): class WanDouHttpProxy(ProxyProvider):
@@ -65,7 +66,7 @@ class WanDouHttpProxy(ProxyProvider):
need_get_count = num - len(ip_cache_list) need_get_count = num - len(ip_cache_list)
self.params.update({"num": min(need_get_count, 100)}) # Maximum 100 self.params.update({"num": min(need_get_count, 100)}) # Maximum 100
ip_infos = [] ip_infos = []
async with httpx.AsyncClient() as client: async with make_async_client() as client:
url = self.api_path + "?" + urlencode(self.params) url = self.api_path + "?" + urlencode(self.params)
utils.logger.info(f"[WanDouHttpProxy.get_proxy] get ip proxy url:{url}") utils.logger.info(f"[WanDouHttpProxy.get_proxy] get ip proxy url:{url}")
response = await client.get( response = await client.get(

3
proxy/proxy_ip_pool.py
View File

@@ -26,6 +26,7 @@ from typing import Dict, List
import httpx import httpx
from tenacity import retry, stop_after_attempt, wait_fixed from tenacity import retry, stop_after_attempt, wait_fixed
from tools.httpx_util import make_async_client
import config import config
from proxy.providers import ( from proxy.providers import (
@@ -81,7 +82,7 @@ class ProxyIpPool:
else: else:
proxy_url = f"http://{proxy.ip}:{proxy.port}" proxy_url = f"http://{proxy.ip}:{proxy.port}"
async with httpx.AsyncClient(proxy=proxy_url) as client: async with make_async_client(proxy=proxy_url) as client:
response = await client.get(self.valid_ip_url) response = await client.get(self.valid_ip_url)
if response.status_code == 200: if response.status_code == 200:
return True return True