jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-07 02:17:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: disable SSL verification for proxy/VPN environments

Browse Source 
Add verify=False to all httpx.AsyncClient calls across bilibili,
weibo, zhihu clients and crawler_util. Fixes SSL certificate
validation errors when running behind a corporate proxy or VPN.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Wei Liu
2026-03-18 12:21:27 +13:00
parent 6742cd598b
commit eb45a6367f
4 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
media_platform/bilibili/client.py
View File

@@ -68,7 +68,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
# Check if proxy has expired before each request # Check if proxy has expired before each request
await self._refresh_proxy_if_expired() await self._refresh_proxy_if_expired()
async with httpx.AsyncClient(proxy=self.proxy) as client: async with httpx.AsyncClient(proxy=self.proxy, verify=False) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs) response = await client.request(method, url, timeout=self.timeout, **kwargs)
try: try:
data: Dict = response.json() data: Dict = response.json()
@@ -222,7 +222,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def get_video_media(self, url: str) -> Union[bytes, None]: async def get_video_media(self, url: str) -> Union[bytes, None]:
# Follow CDN 302 redirects and treat any 2xx as success (some endpoints return 206) # Follow CDN 302 redirects and treat any 2xx as success (some endpoints return 206)
async with httpx.AsyncClient(proxy=self.proxy, follow_redirects=True) as client: async with httpx.AsyncClient(proxy=self.proxy, follow_redirects=True, verify=False) as client:
try: try:
response = await client.request("GET", url, timeout=self.timeout, headers=self.headers) response = await client.request("GET", url, timeout=self.timeout, headers=self.headers)
response.raise_for_status() response.raise_for_status()

6
media_platform/weibo/client.py
View File

@@ -73,7 +73,7 @@ class WeiboClient(ProxyRefreshMixin):
await self._refresh_proxy_if_expired() await self._refresh_proxy_if_expired()
enable_return_response = kwargs.pop("return_response", False) enable_return_response = kwargs.pop("return_response", False)
async with httpx.AsyncClient(proxy=self.proxy) as client: async with httpx.AsyncClient(proxy=self.proxy, verify=False) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs) response = await client.request(method, url, timeout=self.timeout, **kwargs)
if enable_return_response: if enable_return_response:
@@ -261,7 +261,7 @@ class WeiboClient(ProxyRefreshMixin):
:return: :return:
""" """
url = f"{self._host}/detail/{note_id}" url = f"{self._host}/detail/{note_id}"
async with httpx.AsyncClient(proxy=self.proxy) as client: async with httpx.AsyncClient(proxy=self.proxy, verify=False) as client:
response = await client.request("GET", url, timeout=self.timeout, headers=self.headers) response = await client.request("GET", url, timeout=self.timeout, headers=self.headers)
if response.status_code != 200: if response.status_code != 200:
raise DataFetchError(f"get weibo detail err: {response.text}") raise DataFetchError(f"get weibo detail err: {response.text}")
@@ -291,7 +291,7 @@ class WeiboClient(ProxyRefreshMixin):
# Since Weibo images are accessed through i1.wp.com, we need to concatenate the URL # Since Weibo images are accessed through i1.wp.com, we need to concatenate the URL
final_uri = (f"{self._image_agent_host}" final_uri = (f"{self._image_agent_host}"
f"{image_url}") f"{image_url}")
async with httpx.AsyncClient(proxy=self.proxy) as client: async with httpx.AsyncClient(proxy=self.proxy, verify=False) as client:
try: try:
response = await client.request("GET", final_uri, timeout=self.timeout) response = await client.request("GET", final_uri, timeout=self.timeout)
response.raise_for_status() response.raise_for_status()

2
media_platform/zhihu/client.py
View File

@@ -98,7 +98,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
# return response.text # return response.text
return_response = kwargs.pop('return_response', False) return_response = kwargs.pop('return_response', False)
async with httpx.AsyncClient(proxy=self.proxy) as client: async with httpx.AsyncClient(proxy=self.proxy, verify=False) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs) response = await client.request(method, url, timeout=self.timeout, **kwargs)
if response.status_code != 200: if response.status_code != 200:

2
tools/crawler_util.py
View File

@@ -47,7 +47,7 @@ async def find_login_qrcode(page: Page, selector: str) -> str:
) )
login_qrcode_img = str(await elements.get_property("src")) # type: ignore login_qrcode_img = str(await elements.get_property("src")) # type: ignore
if "http://" in login_qrcode_img or "https://" in login_qrcode_img: if "http://" in login_qrcode_img or "https://" in login_qrcode_img:
async with httpx.AsyncClient(follow_redirects=True) as client: async with httpx.AsyncClient(follow_redirects=True, verify=False) as client:
utils.logger.info(f"[find_login_qrcode] get qrcode by url:{login_qrcode_img}") utils.logger.info(f"[find_login_qrcode] get qrcode by url:{login_qrcode_img}")
resp = await client.get(login_qrcode_img, headers={"User-Agent": get_user_agent()}) resp = await client.get(login_qrcode_img, headers={"User-Agent": get_user_agent()})
if resp.status_code == 200: if resp.status_code == 200: