feat: 微博爬虫帖子搜索完成

media_platform/weibo/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2023/12/23 15:40
+# @Desc    :
+from .core import WeiboCrawler
+from .login import WeiboLogin
+from .client import WeiboClient

media_platform/weibo/client.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2023/12/23 15:40
+# @Desc    : 微博爬虫 API 请求 client
+
+import asyncio
+import json
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from urllib.parse import urlencode
+
+import httpx
+from playwright.async_api import BrowserContext, Page
+
+from tools import utils
+
+from .exception import DataFetchError
+from .field import SearchType
+
+
+class WeiboClient:
+    def __init__(
+            self,
+            timeout=10,
+            proxies=None,
+            *,
+            headers: Dict[str, str],
+            playwright_page: Page,
+            cookie_dict: Dict[str, str],
+    ):
+        self.proxies = proxies
+        self.timeout = timeout
+        self.headers = headers
+        self._host = "https://m.weibo.cn"
+        self.playwright_page = playwright_page
+        self.cookie_dict = cookie_dict
+
+    async def request(self, method, url, **kwargs) -> Any:
+        async with httpx.AsyncClient(proxies=self.proxies) as client:
+            response = await client.request(
+                method, url, timeout=self.timeout,
+                **kwargs
+            )
+        data: Dict = response.json()
+        if data.get("ok") != 1:
+            utils.logger.error(f"[WeiboClient.request] request {method}:{url} err, res:{data}")
+            raise DataFetchError(data.get("msg", "unkonw error"))
+        else:
+            return data.get("data", {})
+
+    async def get(self, uri: str, params=None) -> Dict:
+        final_uri = uri
+        if isinstance(params, dict):
+            final_uri = (f"{uri}?"
+                         f"{urlencode(params)}")
+        return await self.request(method="GET", url=f"{self._host}{final_uri}", headers=self.headers)
+
+    async def post(self, uri: str, data: dict) -> Dict:
+        json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
+        return await self.request(method="POST", url=f"{self._host}{uri}",
+                                  data=json_str, headers=self.headers)
+
+    async def pong(self) -> bool:
+        """get a note to check if login state is ok"""
+        utils.logger.info("[WeiboClient.pong] Begin pong weibo...")
+        ping_flag = False
+        try:
+            pass
+        except Exception as e:
+            utils.logger.error(f"[BilibiliClient.pong] Pong weibo failed: {e}, and try to login again...")
+            ping_flag = False
+        return ping_flag
+
+    async def update_cookies(self, browser_context: BrowserContext):
+        cookie_str, cookie_dict = utils.convert_cookies(await browser_context.cookies())
+        self.headers["Cookie"] = cookie_str
+        self.cookie_dict = cookie_dict
+
+    async def get_note_by_keyword(
+            self,
+            keyword: str,
+            page: int = 1,
+            search_type: SearchType = SearchType.DEFAULT
+    ) -> Dict:
+        """
+        search note by keyword
+        :param keyword: 微博搜搜的关键词
+        :param page: 分页参数 -当前页码
+        :param search_type: 搜索的类型，见 weibo/filed.py 中的枚举SearchType
+        :return:
+        """
+        uri = "/api/container/getIndex"
+        containerid = f"100103type={search_type.value}&q={keyword}"
+        params = {
+            "containerid": containerid,
+            "page_type": "searchall",
+            "page": page,
+        }
+        return await self.get(uri, params)

media_platform/weibo/core.py

@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2023/12/23 15:41
+# @Desc    : 微博爬虫主流程代码
+
+
+import asyncio
+import os
+import random
+import time
+from asyncio import Task
+from typing import Dict, List, Optional, Tuple, Union
+
+from playwright.async_api import (BrowserContext, BrowserType, Page,
+                                  async_playwright)
+
+import config
+from base.base_crawler import AbstractCrawler
+from models import weibo
+from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
+from tools import utils
+from var import comment_tasks_var, crawler_type_var
+
+from .client import WeiboClient
+from .exception import DataFetchError
+from .login import WeiboLogin
+from .field import SearchType
+from .help import filter_search_result_card
+
+
+class WeiboCrawler(AbstractCrawler):
+    platform: str
+    login_type: str
+    crawler_type: str
+    context_page: Page
+    wb_client: WeiboClient
+    browser_context: BrowserContext
+
+    def __init__(self):
+        self.index_url = "https://m.weibo.cn"
+        self.user_agent = utils.get_user_agent()
+
+    def init_config(self, platform: str, login_type: str, crawler_type: str):
+        self.platform = platform
+        self.login_type = login_type
+        self.crawler_type = crawler_type
+
+    async def start(self):
+        playwright_proxy_format, httpx_proxy_format = None, None
+        if config.ENABLE_IP_PROXY:
+            ip_proxy_pool = await create_ip_pool(config.IP_PROXY_POOL_COUNT, enable_validate_ip=True)
+            ip_proxy_info: IpInfoModel = await ip_proxy_pool.get_proxy()
+            playwright_proxy_format, httpx_proxy_format = self.format_proxy_info(ip_proxy_info)
+
+        async with async_playwright() as playwright:
+            # Launch a browser context.
+            chromium = playwright.chromium
+            self.browser_context = await self.launch_browser(
+                chromium,
+                None,
+                self.user_agent,
+                headless=config.HEADLESS
+            )
+            # stealth.min.js is a js script to prevent the website from detecting the crawler.
+            await self.browser_context.add_init_script(path="libs/stealth.min.js")
+            self.context_page = await self.browser_context.new_page()
+            await self.context_page.goto(self.index_url)
+
+            # Create a client to interact with the xiaohongshu website.
+            self.wb_client = await self.create_weibo_client(httpx_proxy_format)
+            if not await self.wb_client.pong():
+                login_obj = WeiboLogin(
+                    login_type=self.login_type,
+                    login_phone="",  # your phone number
+                    browser_context=self.browser_context,
+                    context_page=self.context_page,
+                    cookie_str=config.COOKIES
+                )
+                await login_obj.begin()
+                await self.wb_client.update_cookies(browser_context=self.browser_context)
+
+            crawler_type_var.set(self.crawler_type)
+            if self.crawler_type == "search":
+                # Search for video and retrieve their comment information.
+                await self.search()
+            elif self.crawler_type == "detail":
+                # Get the information and comments of the specified post
+                pass
+            else:
+                pass
+            utils.logger.info("[WeiboCrawler.start] Bilibili Crawler finished ...")
+
+    async def search(self):
+        """
+        search weibo note with keywords
+        :return:
+        """
+        utils.logger.info("[WeiboCrawler.search] Begin search weibo keywords")
+        weibo_limit_count = 10
+        for keyword in config.KEYWORDS.split(","):
+            utils.logger.info(f"[WeiboCrawler.search] Current search keyword: {keyword}")
+            page = 1
+            while page * weibo_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
+                search_res = await self.wb_client.get_note_by_keyword(
+                    keyword=keyword,
+                    page=page,
+                    search_type=SearchType.DEFAULT
+                )
+                note_id_list: List[str] = []
+                note_list = filter_search_result_card(search_res.get("cards"))
+                for note_item in note_list:
+                    if note_item :
+                        mblog: Dict = note_item.get("mblog")
+                        note_id_list.append(mblog.get("id"))
+                        await weibo.update_weibo_note(note_item)
+
+                page += 1
+
+    async def create_weibo_client(self, httpx_proxy: Optional[str]) -> WeiboClient:
+        """Create xhs client"""
+        utils.logger.info("[WeiboCrawler.create_weibo_client] Begin create weibo API client ...")
+        cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
+        weibo_client_obj = WeiboClient(
+            proxies=httpx_proxy,
+            headers={
+                "User-Agent": self.user_agent,
+                "Cookie": cookie_str,
+                "Origin": "https://m.weibo.cn",
+                "Referer": "https://m.weibo.cn",
+                "Content-Type": "application/json;charset=UTF-8"
+            },
+            playwright_page=self.context_page,
+            cookie_dict=cookie_dict,
+        )
+        return weibo_client_obj
+
+    @staticmethod
+    def format_proxy_info(ip_proxy_info: IpInfoModel) -> Tuple[Optional[Dict], Optional[Dict]]:
+        """format proxy info for playwright and httpx"""
+        playwright_proxy = {
+            "server": f"{ip_proxy_info.protocol}{ip_proxy_info.ip}:{ip_proxy_info.port}",
+            "username": ip_proxy_info.user,
+            "password": ip_proxy_info.password,
+        }
+        httpx_proxy = {
+            f"{ip_proxy_info.protocol}{ip_proxy_info.ip}": f"{ip_proxy_info.protocol}{ip_proxy_info.user}:{ip_proxy_info.password}@{ip_proxy_info.ip}:{ip_proxy_info.port}"
+        }
+        return playwright_proxy, httpx_proxy
+
+    async def launch_browser(
+            self,
+            chromium: BrowserType,
+            playwright_proxy: Optional[Dict],
+            user_agent: Optional[str],
+            headless: bool = True
+    ) -> BrowserContext:
+        """Launch browser and create browser context"""
+        utils.logger.info("[WeiboCrawler.launch_browser] Begin create browser context ...")
+        if config.SAVE_LOGIN_STATE:
+            user_data_dir = os.path.join(os.getcwd(), "browser_data",
+                                         config.USER_DATA_DIR % self.platform)  # type: ignore
+            browser_context = await chromium.launch_persistent_context(
+                user_data_dir=user_data_dir,
+                accept_downloads=True,
+                headless=headless,
+                proxy=playwright_proxy,  # type: ignore
+                viewport={"width": 1920, "height": 1080},
+                user_agent=user_agent
+            )
+            return browser_context
+        else:
+            browser = await chromium.launch(headless=headless, proxy=playwright_proxy)  # type: ignore
+            browser_context = await browser.new_context(
+                viewport={"width": 1920, "height": 1080},
+                user_agent=user_agent
+            )
+            return browser_context

media_platform/weibo/exception.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2023/12/2 18:44
+# @Desc    :
+
+from httpx import RequestError
+
+
+class DataFetchError(RequestError):
+    """something error when fetch"""
+
+
+class IPBlockError(RequestError):
+    """fetch so fast that the server block us ip"""

media_platform/weibo/field.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2023/12/23 15:41
+# @Desc    :
+from enum import Enum
+
+
+class SearchType(Enum):
+    # 综合
+    DEFAULT = "1"
+
+    # 实时
+    REAL_TIME = "61"
+
+    # 热门
+    POPULAR = "60"
+
+    # 视频
+    VIDEO = "64"

media_platform/weibo/help.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2023/12/24 17:37
+# @Desc    :
+
+from typing import List, Dict
+
+
+def filter_search_result_card(card_list: List[Dict]) -> List[Dict]:
+    """
+    过滤微博搜索的结果，只保留card_type为9类型的数据
+    :param card_list:
+    :return:
+    """
+    note_list: List[Dict] = []
+    for card_item in card_list:
+        if card_item.get("card_type") == 9:
+            note_list.append(card_item)
+        if len(card_item.get("card_group", [])) > 0:
+            card_group = card_item.get("card_group")
+            for card_group_item in card_group:
+                if card_group_item.get("card_type") == 9:
+                    note_list.append(card_group_item)
+
+    return note_list

media_platform/weibo/login.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# @Author  : relakkes@gmail.com
+# @Time    : 2023/12/23 15:42
+# @Desc    : 微博登录实现
+
+import asyncio
+import functools
+import sys
+from typing import Optional
+
+from playwright.async_api import BrowserContext, Page
+from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
+                      wait_fixed)
+
+from base.base_crawler import AbstractLogin
+from tools import utils
+
+
+class WeiboLogin(AbstractLogin):
+    def __init__(self,
+                 login_type: str,
+                 browser_context: BrowserContext,
+                 context_page: Page,
+                 login_phone: Optional[str] = "",
+                 cookie_str: str = ""
+                 ):
+        self.login_type = login_type
+        self.browser_context = browser_context
+        self.context_page = context_page
+        self.login_phone = login_phone
+        self.cookie_str = cookie_str
+
+    async def begin(self):
+        """Start login weibo"""
+        utils.logger.info("[WeiboLogin.begin] Begin login Bilibili ...")
+        if self.login_type == "qrcode":
+            await self.login_by_qrcode()
+        elif self.login_type == "phone":
+            await self.login_by_mobile()
+        elif self.login_type == "cookie":
+            await self.login_by_cookies()
+        else:
+            raise ValueError(
+                "[WeiboLogin.begin] Invalid Login Type Currently only supported qrcode or phone or cookie ...")
+
+    @retry(stop=stop_after_attempt(20), wait=wait_fixed(1), retry=retry_if_result(lambda value: value is False))
+    async def check_login_state(self) -> bool:
+        """
+            Check if the current login status is successful and return True otherwise return False
+            retry decorator will retry 20 times if the return value is False, and the retry interval is 1 second
+            if max retry times reached, raise RetryError
+        """
+        current_cookie = await self.browser_context.cookies()
+        _, cookie_dict = utils.convert_cookies(current_cookie)
+        if cookie_dict.get("SESSDATA", "") or cookie_dict.get("DedeUserID"):
+            return True
+        return False
+
+    async def login_by_qrcode(self):
+        """login weibo website and keep webdriver login state"""
+        utils.logger.info("[WeiboLogin.login_by_qrcode] Begin login weibo by qrcode ...")
+
+        # click login button
+        login_button_ele = self.context_page.locator(
+            "xpath=//div[@class='right-entry__outside go-login-btn']//div"
+        )
+        await login_button_ele.click()
+
+        # find login qrcode
+        qrcode_img_selector = "//div[@class='login-scan-box']//img"
+        base64_qrcode_img = await utils.find_login_qrcode(
+            self.context_page,
+            selector=qrcode_img_selector
+        )
+        if not base64_qrcode_img:
+            utils.logger.info("[WeiboLogin.login_by_qrcode] login failed , have not found qrcode please check ....")
+            sys.exit()
+
+        # show login qrcode
+        partial_show_qrcode = functools.partial(utils.show_qrcode, base64_qrcode_img)
+        asyncio.get_running_loop().run_in_executor(executor=None, func=partial_show_qrcode)
+
+        utils.logger.info(f"[WeiboLogin.login_by_qrcode] Waiting for scan code login, remaining time is 20s")
+        try:
+            await self.check_login_state()
+        except RetryError:
+            utils.logger.info("[WeiboLogin.login_by_qrcode] Login weibo failed by qrcode login method ...")
+            sys.exit()
+
+        wait_redirect_seconds = 5
+        utils.logger.info(
+            f"[WeiboLogin.login_by_qrcode] Login successful then wait for {wait_redirect_seconds} seconds redirect ...")
+        await asyncio.sleep(wait_redirect_seconds)
+
+    async def login_by_mobile(self):
+        pass
+
+    async def login_by_cookies(self):
+        utils.logger.info("[WeiboLogin.login_by_qrcode] Begin login weibo by cookie ...")
+        for key, value in utils.convert_str_cookie_to_dict(self.cookie_str).items():
+            await self.browser_context.add_cookies([{
+                'name': key,
+                'value': value,
+                'domain': ".weibo.cn",
+                'path': "/"
+            }])