jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 19:37:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

升级 httpx 版本至 0.28.1，并修改关键字参数 proxies 至 proxy

Browse Source
This commit is contained in:
未来可欺
2025-07-31 22:48:02 +08:00
parent 9d90e9fc6d
commit 0b81240aed
19 changed files with 204 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
media_platform/bilibili/client.py
View File

@@ -35,13 +35,13 @@ class BilibiliClient(AbstractApiClient):
def __init__( def __init__(
self, self,
timeout=60, # 若开启爬取媒体选项，b 站的长视频需要更久的超时时间 timeout=60, # 若开启爬取媒体选项，b 站的长视频需要更久的超时时间
proxies=None, proxy=None,
*, *,
headers: Dict[str, str], headers: Dict[str, str],
playwright_page: Page, playwright_page: Page,
cookie_dict: Dict[str, str], cookie_dict: Dict[str, str],
): ):
self.proxies = proxies self.proxy = proxy
self.timeout = timeout self.timeout = timeout
self.headers = headers self.headers = headers
self._host = "https://api.bilibili.com" self._host = "https://api.bilibili.com"
@@ -49,7 +49,7 @@ class BilibiliClient(AbstractApiClient):
self.cookie_dict = cookie_dict self.cookie_dict = cookie_dict
async def request(self, method, url, **kwargs) -> Any: async def request(self, method, url, **kwargs) -> Any:
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs) response = await client.request(method, url, timeout=self.timeout, **kwargs)
try: try:
data: Dict = response.json() data: Dict = response.json()
@@ -201,7 +201,7 @@ class BilibiliClient(AbstractApiClient):
return await self.get(uri, params, enable_params_sign=True) return await self.get(uri, params, enable_params_sign=True)
async def get_video_media(self, url: str) -> Union[bytes, None]: async def get_video_media(self, url: str) -> Union[bytes, None]:
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request("GET", url, timeout=self.timeout, headers=self.headers) response = await client.request("GET", url, timeout=self.timeout, headers=self.headers)
if not response.reason_phrase == "OK": if not response.reason_phrase == "OK":
utils.logger.error(f"[BilibiliClient.get_video_media] request {url} err, res:{response.text}") utils.logger.error(f"[BilibiliClient.get_video_media] request {url} err, res:{response.text}")

2
media_platform/bilibili/core.py
View File

@@ -417,7 +417,7 @@ class BilibiliCrawler(AbstractCrawler):
utils.logger.info("[BilibiliCrawler.create_bilibili_client] Begin create bilibili API client ...") utils.logger.info("[BilibiliCrawler.create_bilibili_client] Begin create bilibili API client ...")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies()) cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
bilibili_client_obj = BilibiliClient( bilibili_client_obj = BilibiliClient(
proxies=httpx_proxy, proxy=httpx_proxy,
headers={ headers={
"User-Agent": self.user_agent, "User-Agent": self.user_agent,
"Cookie": cookie_str, "Cookie": cookie_str,

11
media_platform/douyin/client.py
View File

@@ -14,7 +14,7 @@ import json
import urllib.parse import urllib.parse
from typing import Any, Callable, Dict, Union, Optional from typing import Any, Callable, Dict, Union, Optional
import requests import httpx
from playwright.async_api import BrowserContext from playwright.async_api import BrowserContext
from base.base_crawler import AbstractApiClient from base.base_crawler import AbstractApiClient
@@ -31,13 +31,13 @@ class DouYinClient(AbstractApiClient):
def __init__( def __init__(
self, self,
timeout=30, # 若开启爬取媒体选项，抖音的短视频需要更久的超时时间 timeout=30, # 若开启爬取媒体选项，抖音的短视频需要更久的超时时间
proxies=None, proxy=None,
*, *,
headers: Dict, headers: Dict,
playwright_page: Optional[Page], playwright_page: Optional[Page],
cookie_dict: Dict, cookie_dict: Dict,
): ):
self.proxies = proxies self.proxy = proxy
self.timeout = timeout self.timeout = timeout
self.headers = headers self.headers = headers
self._host = "https://www.douyin.com" self._host = "https://www.douyin.com"
@@ -95,7 +95,8 @@ class DouYinClient(AbstractApiClient):
params["a_bogus"] = a_bogus params["a_bogus"] = a_bogus
async def request(self, method, url, **kwargs): async def request(self, method, url, **kwargs):
response = requests.request(method, url, timeout=self.timeout, **kwargs) async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs)
try: try:
if response.text == "" or response.text == "blocked": if response.text == "" or response.text == "blocked":
utils.logger.error(f"request params incrr, response.text: {response.text}") utils.logger.error(f"request params incrr, response.text: {response.text}")
@@ -311,7 +312,7 @@ class DouYinClient(AbstractApiClient):
return result return result
async def get_aweme_media(self, url: str) -> Union[bytes, None]: async def get_aweme_media(self, url: str) -> Union[bytes, None]:
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request("GET", url, timeout=self.timeout, follow_redirects=True) response = await client.request("GET", url, timeout=self.timeout, follow_redirects=True)
if not response.reason_phrase == "OK": if not response.reason_phrase == "OK":
utils.logger.error(f"[DouYinCrawler.get_aweme_media] request {url} err, res:{response.text}") utils.logger.error(f"[DouYinCrawler.get_aweme_media] request {url} err, res:{response.text}")

2
media_platform/douyin/core.py
View File

@@ -237,7 +237,7 @@ class DouYinCrawler(AbstractCrawler):
"""Create douyin client""" """Create douyin client"""
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies()) # type: ignore cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies()) # type: ignore
douyin_client = DouYinClient( douyin_client = DouYinClient(
proxies=httpx_proxy, proxy=httpx_proxy,
headers={ headers={
"User-Agent": await self.context_page.evaluate("() => navigator.userAgent"), "User-Agent": await self.context_page.evaluate("() => navigator.userAgent"),
"Cookie": cookie_str, "Cookie": cookie_str,

6
media_platform/kuaishou/client.py
View File

@@ -30,13 +30,13 @@ class KuaiShouClient(AbstractApiClient):
def __init__( def __init__(
self, self,
timeout=10, timeout=10,
proxies=None, proxy=None,
*, *,
headers: Dict[str, str], headers: Dict[str, str],
playwright_page: Page, playwright_page: Page,
cookie_dict: Dict[str, str], cookie_dict: Dict[str, str],
): ):
self.proxies = proxies self.proxy = proxy
self.timeout = timeout self.timeout = timeout
self.headers = headers self.headers = headers
self._host = "https://www.kuaishou.com/graphql" self._host = "https://www.kuaishou.com/graphql"
@@ -45,7 +45,7 @@ class KuaiShouClient(AbstractApiClient):
self.graphql = KuaiShouGraphQL() self.graphql = KuaiShouGraphQL()
async def request(self, method, url, **kwargs) -> Any: async def request(self, method, url, **kwargs) -> Any:
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs) response = await client.request(method, url, timeout=self.timeout, **kwargs)
data: Dict = response.json() data: Dict = response.json()
if data.get("errors"): if data.get("errors"):

2
media_platform/kuaishou/core.py
View File

@@ -268,7 +268,7 @@ class KuaishouCrawler(AbstractCrawler):
await self.browser_context.cookies() await self.browser_context.cookies()
) )
ks_client_obj = KuaiShouClient( ks_client_obj = KuaiShouClient(
proxies=httpx_proxy, proxy=httpx_proxy,
headers={ headers={
"User-Agent": self.user_agent, "User-Agent": self.user_agent,
"Cookie": cookie_str, "Cookie": cookie_str,

84
media_platform/tieba/client.py
View File

@@ -8,7 +8,6 @@
# 详细许可条款请参阅项目根目录下的LICENSE文件。 # 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
import asyncio import asyncio
import json import json
from typing import Any, Callable, Dict, List, Optional, Union from typing import Any, Callable, Dict, List, Optional, Union
@@ -29,6 +28,7 @@ from .help import TieBaExtractor
class BaiduTieBaClient(AbstractApiClient): class BaiduTieBaClient(AbstractApiClient):
def __init__( def __init__(
self, self,
timeout=10, timeout=10,
@@ -46,7 +46,7 @@ class BaiduTieBaClient(AbstractApiClient):
self.default_ip_proxy = default_ip_proxy self.default_ip_proxy = default_ip_proxy
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) @retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
async def request(self, method, url, return_ori_content=False, proxies=None, **kwargs) -> Union[str, Any]: async def request(self, method, url, return_ori_content=False, proxy=None, **kwargs) -> Union[str, Any]:
""" """
封装httpx的公共请求方法，对请求响应做一些处理 封装httpx的公共请求方法，对请求响应做一些处理
Args: Args:
@@ -59,12 +59,9 @@ class BaiduTieBaClient(AbstractApiClient):
Returns: Returns:
""" """
actual_proxies = proxies if proxies else self.default_ip_proxy actual_proxy = proxy if proxy else self.default_ip_proxy
async with httpx.AsyncClient(proxies=actual_proxies) as client: async with httpx.AsyncClient(proxy=actual_proxy) as client:
response = await client.request( response = await client.request(method, url, timeout=self.timeout, headers=self.headers, **kwargs)
method, url, timeout=self.timeout,
headers=self.headers, **kwargs
)
if response.status_code != 200: if response.status_code != 200:
utils.logger.error(f"Request failed, method: {method}, url: {url}, status code: {response.status_code}") utils.logger.error(f"Request failed, method: {method}, url: {url}, status code: {response.status_code}")
@@ -96,19 +93,14 @@ class BaiduTieBaClient(AbstractApiClient):
final_uri = (f"{uri}?" final_uri = (f"{uri}?"
f"{urlencode(params)}") f"{urlencode(params)}")
try: try:
res = await self.request(method="GET", url=f"{self._host}{final_uri}", res = await self.request(method="GET", url=f"{self._host}{final_uri}", return_ori_content=return_ori_content, **kwargs)
return_ori_content=return_ori_content,
**kwargs)
return res return res
except RetryError as e: except RetryError as e:
if self.ip_pool: if self.ip_pool:
proxie_model = await self.ip_pool.get_proxy() proxie_model = await self.ip_pool.get_proxy()
_, proxies = utils.format_proxy_info(proxie_model) _, proxy = utils.format_proxy_info(proxie_model)
res = await self.request(method="GET", url=f"{self._host}{final_uri}", res = await self.request(method="GET", url=f"{self._host}{final_uri}", return_ori_content=return_ori_content, proxy=proxy, **kwargs)
return_ori_content=return_ori_content, self.default_ip_proxy = proxy
proxies=proxies,
**kwargs)
self.default_ip_proxy = proxies
return res return res
utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，IP已经被Block，请尝试更换新的IP代理: {e}") utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，IP已经被Block，请尝试更换新的IP代理: {e}")
@@ -125,8 +117,7 @@ class BaiduTieBaClient(AbstractApiClient):
""" """
json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False) json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
return await self.request(method="POST", url=f"{self._host}{uri}", return await self.request(method="POST", url=f"{self._host}{uri}", data=json_str, **kwargs)
data=json_str, **kwargs)
async def pong(self) -> bool: async def pong(self) -> bool:
""" """
@@ -161,7 +152,8 @@ class BaiduTieBaClient(AbstractApiClient):
pass pass
async def get_notes_by_keyword( async def get_notes_by_keyword(
self, keyword: str, self,
keyword: str,
page: int = 1, page: int = 1,
page_size: int = 10, page_size: int = 10,
sort: SearchSortType = SearchSortType.TIME_DESC, sort: SearchSortType = SearchSortType.TIME_DESC,
@@ -185,7 +177,7 @@ class BaiduTieBaClient(AbstractApiClient):
"rn": page_size, "rn": page_size,
"pn": page, "pn": page,
"sm": sort.value, "sm": sort.value,
"only_thread": note_type.value "only_thread": note_type.value,
} }
page_content = await self.get(uri, params=params, return_ori_content=True) page_content = await self.get(uri, params=params, return_ori_content=True)
return self._page_extractor.extract_search_note_list(page_content) return self._page_extractor.extract_search_note_list(page_content)
@@ -203,7 +195,10 @@ class BaiduTieBaClient(AbstractApiClient):
page_content = await self.get(uri, return_ori_content=True) page_content = await self.get(uri, return_ori_content=True)
return self._page_extractor.extract_note_detail(page_content) return self._page_extractor.extract_note_detail(page_content)
async def get_note_all_comments(self, note_detail: TiebaNote, crawl_interval: float = 1.0, async def get_note_all_comments(
self,
note_detail: TiebaNote,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None, callback: Optional[Callable] = None,
max_count: int = 10, max_count: int = 10,
) -> List[TiebaComment]: ) -> List[TiebaComment]:
@@ -222,11 +217,10 @@ class BaiduTieBaClient(AbstractApiClient):
current_page = 1 current_page = 1
while note_detail.total_replay_page >= current_page and len(result) < max_count: while note_detail.total_replay_page >= current_page and len(result) < max_count:
params = { params = {
"pn": current_page "pn": current_page,
} }
page_content = await self.get(uri, params=params, return_ori_content=True) page_content = await self.get(uri, params=params, return_ori_content=True)
comments = self._page_extractor.extract_tieba_note_parment_comments(page_content, comments = self._page_extractor.extract_tieba_note_parment_comments(page_content, note_id=note_detail.note_id)
note_id=note_detail.note_id)
if not comments: if not comments:
break break
if len(result) + len(comments) > max_count: if len(result) + len(comments) > max_count:
@@ -240,8 +234,12 @@ class BaiduTieBaClient(AbstractApiClient):
current_page += 1 current_page += 1
return result return result
async def get_comments_all_sub_comments(self, comments: List[TiebaComment], crawl_interval: float = 1.0, async def get_comments_all_sub_comments(
callback: Optional[Callable] = None) -> List[TiebaComment]: self,
comments: List[TiebaComment],
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[TiebaComment]:
""" """
获取指定评论下的所有子评论 获取指定评论下的所有子评论
Args: Args:
@@ -275,8 +273,7 @@ class BaiduTieBaClient(AbstractApiClient):
"pn": current_page # 页码 "pn": current_page # 页码
} }
page_content = await self.get(uri, params=params, return_ori_content=True) page_content = await self.get(uri, params=params, return_ori_content=True)
sub_comments = self._page_extractor.extract_tieba_note_sub_comments(page_content, sub_comments = self._page_extractor.extract_tieba_note_sub_comments(page_content, parent_comment=parment_comment)
parent_comment=parment_comment)
if not sub_comments: if not sub_comments:
break break
@@ -328,17 +325,18 @@ class BaiduTieBaClient(AbstractApiClient):
"un": user_name, "un": user_name,
"pn": page_number, "pn": page_number,
"id": "utf-8", "id": "utf-8",
"_": utils.get_current_timestamp() "_": utils.get_current_timestamp(),
} }
return await self.get(uri, params=params) return await self.get(uri, params=params)
async def get_all_notes_by_creator_user_name(self, async def get_all_notes_by_creator_user_name(
user_name: str, crawl_interval: float = 1.0, self,
user_name: str,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None, callback: Optional[Callable] = None,
max_note_count: int = 0, max_note_count: int = 0,
creator_page_html_content: str = None, creator_page_html_content: str = None,
) -> List[TiebaNote]: ) -> List[TiebaNote]:
""" """
根据创作者用户名获取创作者所有帖子 根据创作者用户名获取创作者所有帖子
Args: Args:
@@ -354,17 +352,9 @@ class BaiduTieBaClient(AbstractApiClient):
# 百度贴吧比较特殊一些，前10个帖子是直接展示在主页上的，要单独处理，通过API获取不到 # 百度贴吧比较特殊一些，前10个帖子是直接展示在主页上的，要单独处理，通过API获取不到
result: List[TiebaNote] = [] result: List[TiebaNote] = []
if creator_page_html_content: if creator_page_html_content:
thread_id_list = ( thread_id_list = (self._page_extractor.extract_tieba_thread_id_list_from_creator_page(creator_page_html_content))
self._page_extractor.extract_tieba_thread_id_list_from_creator_page( utils.logger.info(f"[BaiduTieBaClient.get_all_notes_by_creator] got user_name:{user_name} thread_id_list len : {len(thread_id_list)}")
creator_page_html_content note_detail_task = [self.get_note_by_id(thread_id) for thread_id in thread_id_list]
)
)
utils.logger.info(
f"[BaiduTieBaClient.get_all_notes_by_creator] got user_name:{user_name} thread_id_list len : {len(thread_id_list)}"
)
note_detail_task = [
self.get_note_by_id(thread_id) for thread_id in thread_id_list
]
notes = await asyncio.gather(*note_detail_task) notes = await asyncio.gather(*note_detail_task)
if callback: if callback:
await callback(notes) await callback(notes)
@@ -377,14 +367,12 @@ class BaiduTieBaClient(AbstractApiClient):
while notes_has_more == 1 and (max_note_count == 0 or total_get_count < max_note_count): while notes_has_more == 1 and (max_note_count == 0 or total_get_count < max_note_count):
notes_res = await self.get_notes_by_creator(user_name, page_number) notes_res = await self.get_notes_by_creator(user_name, page_number)
if not notes_res or notes_res.get("no") != 0: if not notes_res or notes_res.get("no") != 0:
utils.logger.error( utils.logger.error(f"[WeiboClient.get_notes_by_creator] got user_name:{user_name} notes failed, notes_res: {notes_res}")
f"[WeiboClient.get_notes_by_creator] got user_name:{user_name} notes failed, notes_res: {notes_res}")
break break
notes_data = notes_res.get("data") notes_data = notes_res.get("data")
notes_has_more = notes_data.get("has_more") notes_has_more = notes_data.get("has_more")
notes = notes_data["thread_list"] notes = notes_data["thread_list"]
utils.logger.info( utils.logger.info(f"[WeiboClient.get_all_notes_by_creator] got user_name:{user_name} notes len : {len(notes)}")
f"[WeiboClient.get_all_notes_by_creator] got user_name:{user_name} notes len : {len(notes)}")
note_detail_task = [self.get_note_by_id(note['thread_id']) for note in notes] note_detail_task = [self.get_note_by_id(note['thread_id']) for note in notes]
notes = await asyncio.gather(*note_detail_task) notes = await asyncio.gather(*note_detail_task)

10
media_platform/weibo/client.py
View File

@@ -36,13 +36,13 @@ class WeiboClient:
def __init__( def __init__(
self, self,
timeout=30, # 若开启爬取媒体选项，weibo 的图片需要更久的超时时间 timeout=30, # 若开启爬取媒体选项，weibo 的图片需要更久的超时时间
proxies=None, proxy=None,
*, *,
headers: Dict[str, str], headers: Dict[str, str],
playwright_page: Page, playwright_page: Page,
cookie_dict: Dict[str, str], cookie_dict: Dict[str, str],
): ):
self.proxies = proxies self.proxy = proxy
self.timeout = timeout self.timeout = timeout
self.headers = headers self.headers = headers
self._host = "https://m.weibo.cn" self._host = "https://m.weibo.cn"
@@ -52,7 +52,7 @@ class WeiboClient:
async def request(self, method, url, **kwargs) -> Union[Response, Dict]: async def request(self, method, url, **kwargs) -> Union[Response, Dict]:
enable_return_response = kwargs.pop("return_response", False) enable_return_response = kwargs.pop("return_response", False)
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs) response = await client.request(method, url, timeout=self.timeout, **kwargs)
if enable_return_response: if enable_return_response:
@@ -217,7 +217,7 @@ class WeiboClient:
:return: :return:
""" """
url = f"{self._host}/detail/{note_id}" url = f"{self._host}/detail/{note_id}"
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request("GET", url, timeout=self.timeout, headers=self.headers) response = await client.request("GET", url, timeout=self.timeout, headers=self.headers)
if response.status_code != 200: if response.status_code != 200:
raise DataFetchError(f"get weibo detail err: {response.text}") raise DataFetchError(f"get weibo detail err: {response.text}")
@@ -247,7 +247,7 @@ class WeiboClient:
# 由于微博图片是通过 i1.wp.com 来访问的，所以需要拼接一下 # 由于微博图片是通过 i1.wp.com 来访问的，所以需要拼接一下
final_uri = (f"{self._image_agent_host}" final_uri = (f"{self._image_agent_host}"
f"{image_url}") f"{image_url}")
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request("GET", final_uri, timeout=self.timeout) response = await client.request("GET", final_uri, timeout=self.timeout)
if not response.reason_phrase == "OK": if not response.reason_phrase == "OK":
utils.logger.error(f"[WeiboClient.get_note_image] request {final_uri} err, res:{response.text}") utils.logger.error(f"[WeiboClient.get_note_image] request {final_uri} err, res:{response.text}")

2
media_platform/weibo/core.py
View File

@@ -289,7 +289,7 @@ class WeiboCrawler(AbstractCrawler):
utils.logger.info("[WeiboCrawler.create_weibo_client] Begin create weibo API client ...") utils.logger.info("[WeiboCrawler.create_weibo_client] Begin create weibo API client ...")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies()) cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
weibo_client_obj = WeiboClient( weibo_client_obj = WeiboClient(
proxies=httpx_proxy, proxy=httpx_proxy,
headers={ headers={
"User-Agent": utils.get_mobile_user_agent(), "User-Agent": utils.get_mobile_user_agent(),
"Cookie": cookie_str, "Cookie": cookie_str,

8
media_platform/xhs/client.py
View File

@@ -33,13 +33,13 @@ class XiaoHongShuClient(AbstractApiClient):
def __init__( def __init__(
self, self,
timeout=30, # 若开启爬取媒体选项，xhs 的长视频需要更久的超时时间 timeout=30, # 若开启爬取媒体选项，xhs 的长视频需要更久的超时时间
proxies=None, proxy=None,
*, *,
headers: Dict[str, str], headers: Dict[str, str],
playwright_page: Page, playwright_page: Page,
cookie_dict: Dict[str, str], cookie_dict: Dict[str, str],
): ):
self.proxies = proxies self.proxy = proxy
self.timeout = timeout self.timeout = timeout
self.headers = headers self.headers = headers
self._host = "https://edith.xiaohongshu.com" self._host = "https://edith.xiaohongshu.com"
@@ -93,7 +93,7 @@ class XiaoHongShuClient(AbstractApiClient):
""" """
# return response.text # return response.text
return_response = kwargs.pop("return_response", False) return_response = kwargs.pop("return_response", False)
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request(method, url, timeout=self.timeout, **kwargs) response = await client.request(method, url, timeout=self.timeout, **kwargs)
if response.status_code == 471 or response.status_code == 461: if response.status_code == 471 or response.status_code == 461:
@@ -151,7 +151,7 @@ class XiaoHongShuClient(AbstractApiClient):
) )
async def get_note_media(self, url: str) -> Union[bytes, None]: async def get_note_media(self, url: str) -> Union[bytes, None]:
async with httpx.AsyncClient(proxies=self.proxies) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request("GET", url, timeout=self.timeout) response = await client.request("GET", url, timeout=self.timeout)
if not response.reason_phrase == "OK": if not response.reason_phrase == "OK":
utils.logger.error(f"[XiaoHongShuClient.get_note_media] request {url} err, res:{response.text}") utils.logger.error(f"[XiaoHongShuClient.get_note_media] request {url} err, res:{response.text}")

2
media_platform/xhs/core.py
View File

@@ -328,7 +328,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
utils.logger.info("[XiaoHongShuCrawler.create_xhs_client] Begin create xiaohongshu API client ...") utils.logger.info("[XiaoHongShuCrawler.create_xhs_client] Begin create xiaohongshu API client ...")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies()) cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
xhs_client_obj = XiaoHongShuClient( xhs_client_obj = XiaoHongShuClient(
proxies=httpx_proxy, proxy=httpx_proxy,
headers={ headers={
"accept": "application/json, text/plain, */*", "accept": "application/json, text/plain, */*",
"accept-language": "zh-CN,zh;q=0.9", "accept-language": "zh-CN,zh;q=0.9",

103
media_platform/zhihu/client.py
View File

@@ -8,7 +8,6 @@
# 详细许可条款请参阅项目根目录下的LICENSE文件。 # 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import asyncio import asyncio
import json import json
@@ -32,16 +31,17 @@ from .help import ZhihuExtractor, sign
class ZhiHuClient(AbstractApiClient): class ZhiHuClient(AbstractApiClient):
def __init__( def __init__(
self, self,
timeout=10, timeout=10,
proxies=None, proxy=None,
*, *,
headers: Dict[str, str], headers: Dict[str, str],
playwright_page: Page, playwright_page: Page,
cookie_dict: Dict[str, str], cookie_dict: Dict[str, str],
): ):
self.proxies = proxies self.proxy = proxy
self.timeout = timeout self.timeout = timeout
self.default_headers = headers self.default_headers = headers
self.cookie_dict = cookie_dict self.cookie_dict = cookie_dict
@@ -79,11 +79,8 @@ class ZhiHuClient(AbstractApiClient):
# return response.text # return response.text
return_response = kwargs.pop('return_response', False) return_response = kwargs.pop('return_response', False)
async with httpx.AsyncClient(proxies=self.proxies, ) as client: async with httpx.AsyncClient(proxy=self.proxy) as client:
response = await client.request( response = await client.request(method, url, timeout=self.timeout, **kwargs)
method, url, timeout=self.timeout,
**kwargs
)
if response.status_code != 200: if response.status_code != 200:
utils.logger.error(f"[ZhiHuClient.request] Requset Url: {url}, Request error: {response.text}") utils.logger.error(f"[ZhiHuClient.request] Requset Url: {url}, Request error: {response.text}")
@@ -106,7 +103,6 @@ class ZhiHuClient(AbstractApiClient):
utils.logger.error(f"[ZhiHuClient.request] Request error: {response.text}") utils.logger.error(f"[ZhiHuClient.request] Request error: {response.text}")
raise DataFetchError(response.text) raise DataFetchError(response.text)
async def get(self, uri: str, params=None, **kwargs) -> Union[Response, Dict, str]: async def get(self, uri: str, params=None, **kwargs) -> Union[Response, Dict, str]:
""" """
GET请求，对请求头签名 GET请求，对请求头签名
@@ -121,11 +117,7 @@ class ZhiHuClient(AbstractApiClient):
if isinstance(params, dict): if isinstance(params, dict):
final_uri += '?' + urlencode(params) final_uri += '?' + urlencode(params)
headers = await self._pre_headers(final_uri) headers = await self._pre_headers(final_uri)
base_url = ( base_url = (zhihu_constant.ZHIHU_URL if "/p/" not in uri else zhihu_constant.ZHIHU_ZHUANLAN_URL)
zhihu_constant.ZHIHU_URL
if "/p/" not in uri
else zhihu_constant.ZHIHU_ZHUANLAN_URL
)
return await self.request(method="GET", url=base_url + final_uri, headers=headers, **kwargs) return await self.request(method="GET", url=base_url + final_uri, headers=headers, **kwargs)
async def pong(self) -> bool: async def pong(self) -> bool:
@@ -167,18 +159,17 @@ class ZhiHuClient(AbstractApiClient):
Returns: Returns:
""" """
params = { params = {"include": "email,is_active,is_bind_phone"}
"include": "email,is_active,is_bind_phone"
}
return await self.get("/api/v4/me", params) return await self.get("/api/v4/me", params)
async def get_note_by_keyword( async def get_note_by_keyword(
self, keyword: str, self,
keyword: str,
page: int = 1, page: int = 1,
page_size: int = 20, page_size: int = 20,
sort: SearchSort = SearchSort.DEFAULT, sort: SearchSort = SearchSort.DEFAULT,
note_type: SearchType = SearchType.DEFAULT, note_type: SearchType = SearchType.DEFAULT,
search_time: SearchTime = SearchTime.DEFAULT search_time: SearchTime = SearchTime.DEFAULT,
) -> List[ZhihuContent]: ) -> List[ZhihuContent]:
""" """
根据关键词搜索 根据关键词搜索
@@ -213,8 +204,14 @@ class ZhiHuClient(AbstractApiClient):
utils.logger.info(f"[ZhiHuClient.get_note_by_keyword] Search result: {search_res}") utils.logger.info(f"[ZhiHuClient.get_note_by_keyword] Search result: {search_res}")
return self._extractor.extract_contents_from_search(search_res) return self._extractor.extract_contents_from_search(search_res)
async def get_root_comments(self, content_id: str, content_type: str, offset: str = "", limit: int = 10, async def get_root_comments(
order_by: str = "score") -> Dict: self,
content_id: str,
content_type: str,
offset: str = "",
limit: int = 10,
order_by: str = "score",
) -> Dict:
""" """
获取内容的一级评论 获取内容的一级评论
Args: Args:
@@ -238,8 +235,13 @@ class ZhiHuClient(AbstractApiClient):
# } # }
# return await self.get(uri, params) # return await self.get(uri, params)
async def get_child_comments(self, root_comment_id: str, offset: str = "", limit: int = 10, async def get_child_comments(
order_by: str = "sort") -> Dict: self,
root_comment_id: str,
offset: str = "",
limit: int = 10,
order_by: str = "sort",
) -> Dict:
""" """
获取一级评论下的子评论 获取一级评论下的子评论
Args: Args:
@@ -255,12 +257,16 @@ class ZhiHuClient(AbstractApiClient):
params = { params = {
"order": order_by, "order": order_by,
"offset": offset, "offset": offset,
"limit": limit "limit": limit,
} }
return await self.get(uri, params) return await self.get(uri, params)
async def get_note_all_comments(self, content: ZhihuContent, crawl_interval: float = 1.0, async def get_note_all_comments(
callback: Optional[Callable] = None) -> List[ZhihuComment]: self,
content: ZhihuContent,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuComment]:
""" """
获取指定帖子下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息 获取指定帖子下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息
Args: Args:
@@ -295,8 +301,13 @@ class ZhiHuClient(AbstractApiClient):
await asyncio.sleep(crawl_interval) await asyncio.sleep(crawl_interval)
return result return result
async def get_comments_all_sub_comments(self, content: ZhihuContent, comments: List[ZhihuComment], crawl_interval: float = 1.0, async def get_comments_all_sub_comments(
callback: Optional[Callable] = None) -> List[ZhihuComment]: self,
content: ZhihuContent,
comments: List[ZhihuComment],
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuComment]:
""" """
获取指定评论下的所有子评论 获取指定评论下的所有子评论
Args: Args:
@@ -365,7 +376,8 @@ class ZhiHuClient(AbstractApiClient):
""" """
uri = f"/api/v4/members/{url_token}/answers" uri = f"/api/v4/members/{url_token}/answers"
params = { params = {
"include":"data[*].is_normal,admin_closed_comment,reward_info,is_collapsed,annotation_action,annotation_detail,collapse_reason,collapsed_by,suggest_edit,comment_count,can_comment,content,editable_content,attachment,voteup_count,reshipment_settings,comment_permission,created_time,updated_time,review_info,excerpt,paid_info,reaction_instruction,is_labeled,label_info,relationship.is_authorized,voting,is_author,is_thanked,is_nothelp;data[*].vessay_info;data[*].author.badge[?(type=best_answerer)].topics;data[*].author.vip_info;data[*].question.has_publishing_draft,relationship", "include":
"data[*].is_normal,admin_closed_comment,reward_info,is_collapsed,annotation_action,annotation_detail,collapse_reason,collapsed_by,suggest_edit,comment_count,can_comment,content,editable_content,attachment,voteup_count,reshipment_settings,comment_permission,created_time,updated_time,review_info,excerpt,paid_info,reaction_instruction,is_labeled,label_info,relationship.is_authorized,voting,is_author,is_thanked,is_nothelp;data[*].vessay_info;data[*].author.badge[?(type=best_answerer)].topics;data[*].author.vip_info;data[*].question.has_publishing_draft,relationship",
"offset": offset, "offset": offset,
"limit": limit, "limit": limit,
"order_by": "created" "order_by": "created"
@@ -385,7 +397,8 @@ class ZhiHuClient(AbstractApiClient):
""" """
uri = f"/api/v4/members/{url_token}/articles" uri = f"/api/v4/members/{url_token}/articles"
params = { params = {
"include":"data[*].comment_count,suggest_edit,is_normal,thumbnail_extra_info,thumbnail,can_comment,comment_permission,admin_closed_comment,content,voteup_count,created,updated,upvoted_followees,voting,review_info,reaction_instruction,is_labeled,label_info;data[*].vessay_info;data[*].author.badge[?(type=best_answerer)].topics;data[*].author.vip_info;", "include":
"data[*].comment_count,suggest_edit,is_normal,thumbnail_extra_info,thumbnail,can_comment,comment_permission,admin_closed_comment,content,voteup_count,created,updated,upvoted_followees,voting,review_info,reaction_instruction,is_labeled,label_info;data[*].vessay_info;data[*].author.badge[?(type=best_answerer)].topics;data[*].author.vip_info;",
"offset": offset, "offset": offset,
"limit": limit, "limit": limit,
"order_by": "created" "order_by": "created"
@@ -405,15 +418,14 @@ class ZhiHuClient(AbstractApiClient):
""" """
uri = f"/api/v4/members/{url_token}/zvideos" uri = f"/api/v4/members/{url_token}/zvideos"
params = { params = {
"include":"similar_zvideo,creation_relationship,reaction_instruction", "include": "similar_zvideo,creation_relationship,reaction_instruction",
"offset": offset, "offset": offset,
"limit": limit, "limit": limit,
"similar_aggregation": "true" "similar_aggregation": "true",
} }
return await self.get(uri, params) return await self.get(uri, params)
async def get_all_anwser_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, async def get_all_anwser_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
callback: Optional[Callable] = None) -> List[ZhihuContent]:
""" """
获取创作者的所有回答 获取创作者的所有回答
Args: Args:
@@ -443,9 +455,12 @@ class ZhiHuClient(AbstractApiClient):
await asyncio.sleep(crawl_interval) await asyncio.sleep(crawl_interval)
return all_contents return all_contents
async def get_all_articles_by_creator(
async def get_all_articles_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, self,
callback: Optional[Callable] = None) -> List[ZhihuContent]: creator: ZhihuCreator,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
""" """
获取创作者的所有文章 获取创作者的所有文章
Args: Args:
@@ -474,9 +489,12 @@ class ZhiHuClient(AbstractApiClient):
await asyncio.sleep(crawl_interval) await asyncio.sleep(crawl_interval)
return all_contents return all_contents
async def get_all_videos_by_creator(
async def get_all_videos_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, self,
callback: Optional[Callable] = None) -> List[ZhihuContent]: creator: ZhihuCreator,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
""" """
获取创作者的所有视频 获取创作者的所有视频
Args: Args:
@@ -505,9 +523,10 @@ class ZhiHuClient(AbstractApiClient):
await asyncio.sleep(crawl_interval) await asyncio.sleep(crawl_interval)
return all_contents return all_contents
async def get_answer_info( async def get_answer_info(
self, question_id: str, answer_id: str self,
question_id: str,
answer_id: str,
) -> Optional[ZhihuContent]: ) -> Optional[ZhihuContent]:
""" """
获取回答信息 获取回答信息

2
media_platform/zhihu/core.py
View File

@@ -360,7 +360,7 @@ class ZhihuCrawler(AbstractCrawler):
await self.browser_context.cookies() await self.browser_context.cookies()
) )
zhihu_client_obj = ZhiHuClient( zhihu_client_obj = ZhiHuClient(
proxies=httpx_proxy, proxy=httpx_proxy,
headers={ headers={
"accept": "*/*", "accept": "*/*",
"accept-language": "zh-CN,zh;q=0.9", "accept-language": "zh-CN,zh;q=0.9",

2
proxy/base_proxy.py
View File

@@ -32,7 +32,7 @@ class IpGetError(Exception):
class ProxyProvider(ABC): class ProxyProvider(ABC):
@abstractmethod @abstractmethod
async def get_proxies(self, num: int) -> List[IpInfoModel]: async def get_proxy(self, num: int) -> List[IpInfoModel]:
""" """
获取 IP 的抽象方法，不同的 HTTP 代理商需要实现该方法 获取 IP 的抽象方法，不同的 HTTP 代理商需要实现该方法
:param num: 提取的 IP 数量 :param num: 提取的 IP 数量

11
proxy/providers/jishu_http_proxy.py
View File

@@ -8,7 +8,6 @@
# 详细许可条款请参阅项目根目录下的LICENSE文件。 # 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com # @Author : relakkes@gmail.com
# @Time : 2024/4/5 09:32 # @Time : 2024/4/5 09:32
@@ -25,6 +24,7 @@ from tools import utils
class JiSuHttpProxy(ProxyProvider): class JiSuHttpProxy(ProxyProvider):
def __init__(self, key: str, crypto: str, time_validity_period: int): def __init__(self, key: str, crypto: str, time_validity_period: int):
""" """
极速HTTP 代理IP实现 极速HTTP 代理IP实现
@@ -44,7 +44,7 @@ class JiSuHttpProxy(ProxyProvider):
} }
self.ip_cache = IpCache() self.ip_cache = IpCache()
async def get_proxies(self, num: int) -> List[IpInfoModel]: async def get_proxy(self, num: int) -> List[IpInfoModel]:
""" """
:param num: :param num:
:return: :return:
@@ -61,9 +61,10 @@ class JiSuHttpProxy(ProxyProvider):
ip_infos = [] ip_infos = []
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
url = self.api_path + "/fetchips" + '?' + urlencode(self.params) url = self.api_path + "/fetchips" + '?' + urlencode(self.params)
utils.logger.info(f"[JiSuHttpProxy.get_proxies] get ip proxy url:{url}") utils.logger.info(f"[JiSuHttpProxy.get_proxy] get ip proxy url:{url}")
response = await client.get(url, headers={ response = await client.get(url, headers={
"User-Agent": "MediaCrawler https://github.com/NanmiCoder/MediaCrawler"}) "User-Agent": "MediaCrawler https://github.com/NanmiCoder/MediaCrawler",
})
res_dict: Dict = response.json() res_dict: Dict = response.json()
if res_dict.get("code") == 0: if res_dict.get("code") == 0:
data: List[Dict] = res_dict.get("data") data: List[Dict] = res_dict.get("data")
@@ -74,7 +75,7 @@ class JiSuHttpProxy(ProxyProvider):
port=ip_item.get("port"), port=ip_item.get("port"),
user=ip_item.get("user"), user=ip_item.get("user"),
password=ip_item.get("pass"), password=ip_item.get("pass"),
expired_time_ts=utils.get_unix_time_from_time_str(ip_item.get("expire")) expired_time_ts=utils.get_unix_time_from_time_str(ip_item.get("expire")),
) )
ip_key = f"JISUHTTP_{ip_info_model.ip}_{ip_info_model.port}_{ip_info_model.user}_{ip_info_model.password}" ip_key = f"JISUHTTP_{ip_info_model.ip}_{ip_info_model.port}_{ip_info_model.user}_{ip_info_model.password}"
ip_value = ip_info_model.json() ip_value = ip_info_model.json()

2
proxy/providers/kuaidl_proxy.py
View File

@@ -80,7 +80,7 @@ class KuaiDaiLiProxy(ProxyProvider):
"f_et": 1, "f_et": 1,
} }
async def get_proxies(self, num: int) -> List[IpInfoModel]: async def get_proxy(self, num: int) -> List[IpInfoModel]:
""" """
快代理实现 快代理实现
Args: Args:

15
proxy/proxy_ip_pool.py
View File

@@ -8,7 +8,6 @@
# 详细许可条款请参阅项目根目录下的LICENSE文件。 # 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com # @Author : relakkes@gmail.com
# @Time : 2023/12/2 13:45 # @Time : 2023/12/2 13:45
@@ -28,6 +27,7 @@ from .types import IpInfoModel, ProviderNameEnum
class ProxyIpPool: class ProxyIpPool:
def __init__(self, ip_pool_count: int, enable_validate_ip: bool, ip_provider: ProxyProvider) -> None: def __init__(self, ip_pool_count: int, enable_validate_ip: bool, ip_provider: ProxyProvider) -> None:
""" """
@@ -48,7 +48,7 @@ class ProxyIpPool:
Returns: Returns:
""" """
self.proxy_list = await self.ip_provider.get_proxies(self.ip_pool_count) self.proxy_list = await self.ip_provider.get_proxy(self.ip_pool_count)
async def _is_valid_proxy(self, proxy: IpInfoModel) -> bool: async def _is_valid_proxy(self, proxy: IpInfoModel) -> bool:
""" """
@@ -59,9 +59,9 @@ class ProxyIpPool:
utils.logger.info(f"[ProxyIpPool._is_valid_proxy] testing {proxy.ip} is it valid ") utils.logger.info(f"[ProxyIpPool._is_valid_proxy] testing {proxy.ip} is it valid ")
try: try:
httpx_proxy = { httpx_proxy = {
f"{proxy.protocol}": f"http://{proxy.user}:{proxy.password}@{proxy.ip}:{proxy.port}" f"{proxy.protocol}": f"http://{proxy.user}:{proxy.password}@{proxy.ip}:{proxy.port}",
} }
async with httpx.AsyncClient(proxies=httpx_proxy) as client: async with httpx.AsyncClient(proxy=httpx_proxy) as client:
response = await client.get(self.valid_ip_url) response = await client.get(self.valid_ip_url)
if response.status_code == 200: if response.status_code == 200:
return True return True
@@ -98,7 +98,7 @@ class ProxyIpPool:
IpProxyProvider: Dict[str, ProxyProvider] = { IpProxyProvider: Dict[str, ProxyProvider] = {
ProviderNameEnum.JISHU_HTTP_PROVIDER.value: new_jisu_http_proxy(), ProviderNameEnum.JISHU_HTTP_PROVIDER.value: new_jisu_http_proxy(),
ProviderNameEnum.KUAI_DAILI_PROVIDER.value: new_kuai_daili_proxy() ProviderNameEnum.KUAI_DAILI_PROVIDER.value: new_kuai_daili_proxy(),
} }
@@ -109,9 +109,10 @@ async def create_ip_pool(ip_pool_count: int, enable_validate_ip: bool) -> ProxyI
:param enable_validate_ip: 是否开启验证IP代理 :param enable_validate_ip: 是否开启验证IP代理
:return: :return:
""" """
pool = ProxyIpPool(ip_pool_count=ip_pool_count, pool = ProxyIpPool(
ip_pool_count=ip_pool_count,
enable_validate_ip=enable_validate_ip, enable_validate_ip=enable_validate_ip,
ip_provider=IpProxyProvider.get(config.IP_PROXY_PROVIDER_NAME) ip_provider=IpProxyProvider.get(config.IP_PROXY_PROVIDER_NAME),
) )
await pool.load_proxies() await pool.load_proxies()
return pool return pool

2
pyproject.toml
View File

@@ -10,7 +10,7 @@ dependencies = [
"aiomysql==0.2.0", "aiomysql==0.2.0",
"aiosqlite>=0.21.0", "aiosqlite>=0.21.0",
"fastapi==0.110.2", "fastapi==0.110.2",
"httpx==0.24.0", "httpx==0.28.1",
"jieba==0.42.1", "jieba==0.42.1",
"matplotlib==3.9.0", "matplotlib==3.9.0",
"opencv-python>=4.11.0.86", "opencv-python>=4.11.0.86",

2
requirements.txt
View File

@@ -1,4 +1,4 @@
httpx==0.24.0 httpx==0.28.1
Pillow==9.5.0 Pillow==9.5.0
playwright==1.45.0 playwright==1.45.0
tenacity==8.2.2 tenacity==8.2.2