feat: 增加配置项支持自由选择数据是否保存到关系型数据库中

This commit is contained in:
Relakkes
2023-07-24 20:59:43 +08:00
parent 745e59c875
commit e75707443b
20 changed files with 339 additions and 169 deletions

View File

@@ -83,12 +83,15 @@ class XHSClient:
async def ping(self) -> bool:
"""get a note to check if login state is ok"""
utils.logger.info("begin to ping xhs...")
note_id = "5e5cb38a000000000100185e"
ping_flag = False
try:
note_card: Dict = await self.get_note_by_id(note_id)
return note_card.get("note_id") == note_id
except Exception:
return False
note_card: Dict = await self.get_note_by_keyword(keyword="小红书")
if note_card.get("items"):
ping_flag = True
except Exception as e:
utils.logger.error(f"ping xhs failed: {e}")
ping_flag = False
return ping_flag
async def update_cookies(self, browser_context: BrowserContext):
cookie_str, cookie_dict = utils.convert_cookies(await browser_context.cookies())

View File

@@ -15,7 +15,7 @@ from tools import utils
from .exception import *
from .login import XHSLogin
from .client import XHSClient
from models import xhs as xhs_model
from models import xiaohongshu as xhs_model
from base.base_crawler import AbstractCrawler
from base.proxy_account_pool import AccountPool