jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

原始的HTTPStatusError不能捕获像ConnectError、ReadError这些异常类型，本次提交修改了捕获异常的类型为httpx模块请求异常的基类：HTTPError，以便捕获在httpx.request方法中引发的任何异常（例如ip被封，服务器拒接连接），正确处理爬取媒体被中断时并不会导致爬取文本的中断逻辑

Browse Source
This commit is contained in:
未来可欺
2025-08-06 11:24:51 +08:00
parent 81f2dbe4ab
commit 6a10d0d11c
4 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
media_platform/bilibili/client.py
View File

@@ -210,8 +210,8 @@ class BilibiliClient(AbstractApiClient):
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[BilibiliClient.get_video_media] {exc}") utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None
async def get_video_comments( async def get_video_comments(

4
media_platform/douyin/client.py
View File

@@ -321,6 +321,6 @@ class DouYinClient(AbstractApiClient):
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc}") utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None

4
media_platform/weibo/client.py
View File

@@ -256,8 +256,8 @@ class WeiboClient:
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc}") utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None
async def get_creator_container_info(self, creator_id: str) -> Dict: async def get_creator_container_info(self, creator_id: str) -> Dict:

4
media_platform/xhs/client.py
View File

@@ -160,8 +160,8 @@ class XiaoHongShuClient(AbstractApiClient):
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc}") utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None
async def pong(self) -> bool: async def pong(self) -> bool: