jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-08 19:07:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 完善类型注释，增加 mypy 类型检测

Browse Source
This commit is contained in:
Nanmi
2023-07-16 17:57:18 +08:00
parent e5bdc63323
commit 745e59c875
18 changed files with 116 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
media_platform/douyin/client.py
View File

@@ -18,9 +18,10 @@ class DOUYINClient:
self,
timeout=30,
proxies=None,
headers: Optional[Dict] = None,
playwright_page: Page = None,
cookie_dict: Dict = None
*,
headers: Dict,
playwright_page: Optional[Page],
cookie_dict: Dict
):
self.proxies = proxies
self.timeout = timeout
@@ -33,7 +34,7 @@ class DOUYINClient:
if not params:
return
headers = headers or self.headers
local_storage: Dict = await self.playwright_page.evaluate("() => window.localStorage")
local_storage: Dict = await self.playwright_page.evaluate("() => window.localStorage") # type: ignore
douyin_js_obj = execjs.compile(open('libs/douyin.js').read())
common_params = {
"device_platform": "webapp",
@@ -141,7 +142,7 @@ class DOUYINClient:
del headers["Origin"]
return await self.get("/aweme/v1/web/aweme/detail/", params, headers)
async def get_aweme_comments(self, aweme_id: str, cursor: str = ""):
async def get_aweme_comments(self, aweme_id: str, cursor: int = 0):
"""get note comments
"""

35
media_platform/douyin/core.py
View File

@@ -20,20 +20,21 @@ from models import douyin
class DouYinCrawler(AbstractCrawler):
def __init__(self):
self.browser_context: Optional[BrowserContext] = None
self.context_page: Optional[Page] = None
dy_client: DOUYINClient
def __init__(self) -> None:
self.browser_context: Optional[BrowserContext] = None # type: ignore
self.context_page: Optional[Page] = None # type: ignore
self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" # fixed
self.dy_client: Optional[DOUYINClient] = None
self.index_url = "https://www.douyin.com"
self.command_args: Optional[Namespace] = None
self.account_pool: Optional[AccountPool] = None
self.command_args: Optional[Namespace] = None # type: ignore
self.account_pool: Optional[AccountPool] = None # type: ignore
def init_config(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
async def start(self):
async def start(self) -> None:
account_phone, playwright_proxy, httpx_proxy = self.create_proxy_info()
async with async_playwright() as playwright:
# Launch a browser context.
@@ -52,7 +53,7 @@ class DouYinCrawler(AbstractCrawler):
self.dy_client = await self.create_douyin_client(httpx_proxy)
if not await self.dy_client.ping(browser_context=self.browser_context):
login_obj = DouYinLogin(
login_type=self.command_args.lt,
login_type=self.command_args.lt, # type: ignore
login_phone=account_phone,
browser_context=self.browser_context,
context_page=self.context_page,
@@ -66,7 +67,7 @@ class DouYinCrawler(AbstractCrawler):
utils.logger.info("Douyin Crawler finished ...")
async def search_posts(self):
async def search_posts(self) -> None:
utils.logger.info("Begin search douyin keywords")
for keyword in config.KEYWORDS.split(","):
utils.logger.info(f"Current keyword: {keyword}")
@@ -87,7 +88,7 @@ class DouYinCrawler(AbstractCrawler):
post_item.get("aweme_mix_info", {}).get("mix_items")[0]
except TypeError:
continue
aweme_list.append(aweme_info.get("aweme_id"))
aweme_list.append(aweme_info.get("aweme_id",""))
await douyin.update_douyin_aweme(aweme_item=aweme_info)
utils.logger.info(f"keyword:{keyword}, aweme_list:{aweme_list}")
# await self.batch_get_note_comments(aweme_list)
@@ -115,7 +116,7 @@ class DouYinCrawler(AbstractCrawler):
return None, None, None
# phone: 13012345671 ip_proxy: 111.122.xx.xx1:8888
phone, ip_proxy = self.account_pool.get_account()
phone, ip_proxy = self.account_pool.get_account() # type: ignore
playwright_proxy = {
"server": f"{config.IP_PROXY_PROTOCOL}{ip_proxy}",
"username": config.IP_PROXY_USER,
@@ -124,9 +125,9 @@ class DouYinCrawler(AbstractCrawler):
httpx_proxy = f"{config.IP_PROXY_PROTOCOL}{config.IP_PROXY_USER}:{config.IP_PROXY_PASSWORD}@{ip_proxy}"
return phone, playwright_proxy, httpx_proxy
async def create_douyin_client(self, httpx_proxy: str) -> DOUYINClient:
async def create_douyin_client(self, httpx_proxy: Optional[str]) -> DOUYINClient:
"""Create douyin client"""
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies()) # type: ignore
douyin_client = DOUYINClient(
proxies=httpx_proxy,
headers={
@@ -151,18 +152,18 @@ class DouYinCrawler(AbstractCrawler):
) -> BrowserContext:
"""Launch browser and create browser context"""
if config.SAVE_LOGIN_STATE:
user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % self.command_args.platform)
user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % self.command_args.platform) # type: ignore
browser_context = await chromium.launch_persistent_context(
user_data_dir=user_data_dir,
accept_downloads=True,
headless=headless,
proxy=playwright_proxy,
proxy=playwright_proxy, # type: ignore
viewport={"width": 1920, "height": 1080},
user_agent=user_agent
)
) # type: ignore
return browser_context
else:
browser = await chromium.launch(headless=headless, proxy=playwright_proxy)
browser = await chromium.launch(headless=headless, proxy=playwright_proxy) # type: ignore
browser_context = await browser.new_context(
viewport={"width": 1920, "height": 1080},
user_agent=user_agent

23
media_platform/douyin/login.py
View File

@@ -1,6 +1,7 @@
import sys
import asyncio
import functools
from typing import Optional
import aioredis
from tenacity import (
@@ -22,10 +23,10 @@ class DouYinLogin(AbstractLogin):
def __init__(self,
login_type: str,
browser_context: BrowserContext,
context_page: Page,
login_phone: str = None,
cookie_str: str = None
browser_context: BrowserContext, # type: ignore
context_page: Page, # type: ignore
login_phone: Optional[str] = "",
cookie_str: Optional[str] = ""
):
self.login_type = login_type
self.browser_context = browser_context
@@ -202,14 +203,14 @@ class DouYinLogin(AbstractLogin):
selector=back_selector,
timeout=1000 * 10, # wait 10 seconds
)
slide_back = str(await slider_back_elements.get_property("src"))
slide_back = str(await slider_back_elements.get_property("src")) # type: ignore
# get slider gap image
gap_elements = await self.context_page.wait_for_selector(
selector=gap_selector,
timeout=1000 * 10, # wait 10 seconds
)
gap_src = str(await gap_elements.get_property("src"))
gap_src = str(await gap_elements.get_property("src")) # type: ignore
# 识别滑块位置
slide_app = utils.Slide(gap=gap_src, bg=slide_back)
@@ -223,14 +224,14 @@ class DouYinLogin(AbstractLogin):
# 根据轨迹拖拽滑块到指定位置
element = await self.context_page.query_selector(gap_selector)
bounding_box = await element.bounding_box()
bounding_box = await element.bounding_box() # type: ignore
await self.context_page.mouse.move(bounding_box["x"] + bounding_box["width"] / 2,
bounding_box["y"] + bounding_box["height"] / 2)
await self.context_page.mouse.move(bounding_box["x"] + bounding_box["width"] / 2, # type: ignore
bounding_box["y"] + bounding_box["height"] / 2) # type: ignore
# 这里获取到x坐标中心点位置
x = bounding_box["x"] + bounding_box["width"] / 2
x = bounding_box["x"] + bounding_box["width"] / 2 # type: ignore
# 模拟滑动操作
await element.hover()
await element.hover() # type: ignore
await self.context_page.mouse.down()
for track in tracks:

22
media_platform/xhs/client.py
View File

@@ -17,9 +17,10 @@ class XHSClient:
self,
timeout=10,
proxies=None,
headers: Optional[Dict] = None,
playwright_page: Page = None,
cookie_dict: Dict = None
*,
headers: Dict[str, str],
playwright_page: Page,
cookie_dict: Dict[str, str],
):
self.proxies = proxies
self.timeout = timeout
@@ -51,21 +52,21 @@ class XHSClient:
self.headers.update(headers)
return self.headers
async def request(self, method, url, **kwargs):
async def request(self, method, url, **kwargs) -> Dict:
async with httpx.AsyncClient(proxies=self.proxies) as client:
response = await client.request(
method, url, timeout=self.timeout,
**kwargs
)
data = response.json()
data: Dict = response.json()
if data["success"]:
return data.get("data", data.get("success"))
return data.get("data", data.get("success", {}))
elif data["code"] == self.IP_ERROR_CODE:
raise IPBlockError(self.IP_ERROR_STR)
else:
raise DataFetchError(data.get("msg", None))
async def get(self, uri: str, params=None):
async def get(self, uri: str, params=None) -> Dict:
final_uri = uri
if isinstance(params, dict):
final_uri = (f"{uri}?"
@@ -73,7 +74,7 @@ class XHSClient:
headers = await self._pre_headers(final_uri)
return await self.request(method="GET", url=f"{self._host}{final_uri}", headers=headers)
async def post(self, uri: str, data: dict):
async def post(self, uri: str, data: dict) -> Dict:
headers = await self._pre_headers(uri, data)
json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
return await self.request(method="POST", url=f"{self._host}{uri}",
@@ -86,7 +87,7 @@ class XHSClient:
try:
note_card: Dict = await self.get_note_by_id(note_id)
return note_card.get("note_id") == note_id
except DataFetchError:
except Exception:
return False
async def update_cookies(self, browser_context: BrowserContext):
@@ -128,7 +129,8 @@ class XHSClient:
data = {"source_note_id": note_id}
uri = "/api/sns/web/v1/feed"
res = await self.post(uri, data)
return res["items"][0]["note_card"]
res_dict: Dict = res["items"][0]["note_card"]
return res_dict
async def get_note_comments(self, note_id: str, cursor: str = "") -> Dict:
"""get note comments

22
media_platform/xhs/core.py
View File

@@ -21,15 +21,15 @@ from base.proxy_account_pool import AccountPool
class XiaoHongShuCrawler(AbstractCrawler):
context_page: Page
browser_context: BrowserContext
xhs_client: XHSClient
account_pool: AccountPool
def __init__(self):
self.browser_context: Optional[BrowserContext] = None
self.context_page: Optional[Page] = None
self.user_agent = utils.get_user_agent()
self.xhs_client: Optional[XHSClient] = None
self.index_url = "https://www.xiaohongshu.com"
self.command_args: Optional[Namespace] = None
self.account_pool: Optional[AccountPool] = None
self.command_args: Optional[Namespace] = None # type: ignore
self.user_agent = utils.get_user_agent()
def init_config(self, **kwargs):
for key, value in kwargs.items():
@@ -69,7 +69,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
utils.logger.info("Xhs Crawler finished ...")
async def search_posts(self):
async def search_posts(self) -> None:
"""Search for notes and retrieve their comment information."""
utils.logger.info("Begin search xiaohongshu keywords")
@@ -86,7 +86,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
_semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_note_detail(post_item.get("id"), _semaphore)
for post_item in posts_res.get("items")
for post_item in posts_res.get("items", {})
]
note_details = await asyncio.gather(*task_list)
for note_detail in note_details:
@@ -170,18 +170,18 @@ class XiaoHongShuCrawler(AbstractCrawler):
if config.SAVE_LOGIN_STATE:
# feat issue #14
# we will save login state to avoid login every time
user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % self.command_args.platform)
user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % self.command_args.platform) # type: ignore
browser_context = await chromium.launch_persistent_context(
user_data_dir=user_data_dir,
accept_downloads=True,
headless=headless,
proxy=playwright_proxy,
proxy=playwright_proxy, # type: ignore
viewport={"width": 1920, "height": 1080},
user_agent=user_agent
)
return browser_context
else:
browser = await chromium.launch(headless=headless, proxy=playwright_proxy)
browser = await chromium.launch(headless=headless, proxy=playwright_proxy) # type: ignore
browser_context = await browser.new_context(
viewport={"width": 1920, "height": 1080},
user_agent=user_agent

2
media_platform/xhs/help.py
View File

@@ -92,7 +92,7 @@ def mrc(e):
]
o = -1
def right_without_sign(num, bit=0) -> int:
def right_without_sign(num: int, bit: int=0) -> int:
val = ctypes.c_uint32(num).value >> bit
MAX32INT = 4294967295
return (val + (MAX32INT + 1)) % (2 * (MAX32INT + 1)) - MAX32INT - 1

4
media_platform/xhs/login.py
View File

@@ -24,8 +24,8 @@ class XHSLogin(AbstractLogin):
login_type: str,
browser_context: BrowserContext,
context_page: Page,
login_phone: str = None,
cookie_str: str = None
login_phone: str = "",
cookie_str: str = ""
):
self.login_type = login_type
self.browser_context = browser_context