mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-05-25 20:17:30 +08:00
fix: make SSL verification opt-in via config, extend fix to all platforms
- Add DISABLE_SSL_VERIFY = False to base_config.py (default: verification on) - Add tools/httpx_util.py with make_async_client() factory that reads the config - Replace all httpx.AsyncClient() call sites across all platforms (bilibili, weibo, zhihu, xhs, douyin, kuaishou) and crawler_util with make_async_client() - Extends SSL fix to previously missed platforms: xhs, douyin, kuaishou Users running behind an intercepting proxy can set DISABLE_SSL_VERIFY = True in config/base_config.py. All other users retain certificate verification. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -31,6 +31,7 @@ import config
|
||||
from base.base_crawler import AbstractApiClient
|
||||
from proxy.proxy_mixin import ProxyRefreshMixin
|
||||
from tools import utils
|
||||
from tools.httpx_util import make_async_client
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from proxy.proxy_ip_pool import ProxyIpPool
|
||||
@@ -65,7 +66,7 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
# Check if proxy is expired before each request
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
async with httpx.AsyncClient(proxy=self.proxy) as client:
|
||||
async with make_async_client(proxy=self.proxy) as client:
|
||||
response = await client.request(method, url, timeout=self.timeout, **kwargs)
|
||||
data: Dict = response.json()
|
||||
if data.get("errors"):
|
||||
@@ -97,7 +98,7 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
||||
async with httpx.AsyncClient(proxy=self.proxy) as client:
|
||||
async with make_async_client(proxy=self.proxy) as client:
|
||||
response = await client.request(
|
||||
method="POST",
|
||||
url=f"{self._rest_host}{uri}",
|
||||
|
||||
Reference in New Issue
Block a user