refactor: weibo login

media_platform/weibo/client.py

@@ -23,6 +23,7 @@ from urllib.parse import parse_qs, unquote, urlencode
 import httpx
 from httpx import Response
 from playwright.async_api import BrowserContext, Page
+from tenacity import retry, stop_after_attempt, wait_fixed
 
 import config
 from tools import utils
@@ -50,6 +51,7 @@ class WeiboClient:
         self.cookie_dict = cookie_dict
         self._image_agent_host = "https://i1.wp.com/"
 
+    @retry(stop=stop_after_attempt(5), wait=wait_fixed(2))
     async def request(self, method, url, **kwargs) -> Union[Response, Dict]:
         enable_return_response = kwargs.pop("return_response", False)
         async with httpx.AsyncClient(proxy=self.proxy) as client:
@@ -58,7 +60,12 @@ class WeiboClient:
         if enable_return_response:
             return response
 
-        data: Dict = response.json()
+        try:
+            data: Dict = response.json()
+        except json.decoder.JSONDecodeError:
+            utils.logger.error(f"[WeiboClient.request] request {method}:{url} err code: {response.status_code} res:{response.text}")
+            raise DataFetchError(f"get response code error: {response.status_code}")
+
         ok_code = data.get("ok")
         if ok_code == 0:  # response error
             utils.logger.error(f"[WeiboClient.request] request {method}:{url} err, res:{data}")
@@ -99,10 +106,24 @@ class WeiboClient:
             ping_flag = False
         return ping_flag
 
-    async def update_cookies(self, browser_context: BrowserContext):
-        cookie_str, cookie_dict = utils.convert_cookies(await browser_context.cookies())
+    async def update_cookies(self, browser_context: BrowserContext, urls: Optional[List[str]] = None):
+        """
+        Update cookies from browser context
+        :param browser_context: Browser context
+        :param urls: Optional list of URLs to filter cookies (e.g., ["https://m.weibo.cn"])
+                     If provided, only cookies for these URLs will be retrieved
+        """
+        if urls:
+            cookies = await browser_context.cookies(urls=urls)
+            utils.logger.info(f"[WeiboClient.update_cookies] Updating cookies for specific URLs: {urls}")
+        else:
+            cookies = await browser_context.cookies()
+            utils.logger.info("[WeiboClient.update_cookies] Updating all cookies")
+
+        cookie_str, cookie_dict = utils.convert_cookies(cookies)
         self.headers["Cookie"] = cookie_str
         self.cookie_dict = cookie_dict
+        utils.logger.info(f"[WeiboClient.update_cookies] Cookie updated successfully, total: {len(cookie_dict)} cookies")
 
     async def get_note_by_keyword(
         self,

media_platform/weibo/core.py

@@ -83,7 +83,8 @@ class WeiboCrawler(AbstractCrawler):
 
 
             self.context_page = await self.browser_context.new_page()
-            await self.context_page.goto(self.mobile_index_url)
+            await self.context_page.goto(self.index_url)
+            await asyncio.sleep(2)
 
             # Create a client to interact with the xiaohongshu website.
             self.wb_client = await self.create_weibo_client(httpx_proxy_format)
@@ -100,8 +101,12 @@ class WeiboCrawler(AbstractCrawler):
                 # 登录成功后重定向到手机端的网站，再更新手机端登录成功的cookie
                 utils.logger.info("[WeiboCrawler.start] redirect weibo mobile homepage and update cookies on mobile platform")
                 await self.context_page.goto(self.mobile_index_url)
-                await asyncio.sleep(2)
-                await self.wb_client.update_cookies(browser_context=self.browser_context)
+                await asyncio.sleep(3)
+                # 只获取移动端的 cookies，避免 PC 端和移动端 cookies 混淆
+                await self.wb_client.update_cookies(
+                    browser_context=self.browser_context,
+                    urls=[self.mobile_index_url]
+                )
 
             crawler_type_var.set(config.CRAWLER_TYPE)
             if config.CRAWLER_TYPE == "search":