jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 03:17:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 贴吧搜索重构

Browse Source
This commit is contained in:
Relakkes
2024-08-07 01:01:21 +08:00
parent 1b585cb215
commit 3c98808409
17 changed files with 1057 additions and 7707 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
media_platform/tieba/core.py
View File

@@ -9,7 +9,8 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
import config
from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool, ProxyIpPool
from model.m_baidu_tieba import TiebaNote
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import tieba as tieba_store
from tools import utils
from tools.crawler_util import format_proxy_info
@@ -66,8 +67,7 @@ class TieBaCrawler(AbstractCrawler):
Returns:
"""
utils.logger.info("[BaiduTieBaCrawler.search] Begin search baidutieba keywords")
utils.logger.info("[BaiduTieBaCrawler.search] Begin search baidu tieba keywords")
tieba_limit_count = 10 # tieba limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < tieba_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = tieba_limit_count
@@ -82,52 +82,36 @@ class TieBaCrawler(AbstractCrawler):
continue
try:
utils.logger.info(f"[BaiduTieBaCrawler.search] search tieba keyword: {keyword}, page: {page}")
note_id_list: List[str] = []
notes_list_res = await self.tieba_client.get_notes_by_keyword(
notes_list: List[TiebaNote] = await self.tieba_client.get_notes_by_keyword(
keyword=keyword,
page=page,
page_size=tieba_limit_count,
sort=SearchSortType.TIME_DESC,
note_type=SearchNoteType.FIXED_THREAD
)
utils.logger.info(f"[BaiduTieBaCrawler.search] Search notes res:{notes_list_res}")
if not notes_list_res:
if not notes_list:
utils.logger.info(f"[BaiduTieBaCrawler.search] Search note list is empty")
break
for note_detail in notes_list_res:
if note_detail:
await tieba_store.update_tieba_note(note_detail)
note_id_list.append(note_detail.get("note_id"))
utils.logger.info(f"[BaiduTieBaCrawler.search] Note List: {notes_list}")
await self.get_specified_notes(note_id_list=[note_detail.note_id for note_detail in notes_list])
page += 1
utils.logger.info(f"[BaiduTieBaCrawler.search] Note details: {notes_list_res}")
await self.batch_get_note_comments(note_id_list)
except Exception as ex:
utils.logger.error(f"[BaiduTieBaCrawler.search] Search note list error, err: {ex}")
utils.logger.error(
f"[BaiduTieBaCrawler.search] Search keywords error, current page: {page}, current keyword: {keyword}, err: {ex}")
break
async def fetch_creator_notes_detail(self, note_list: List[Dict]):
async def get_specified_notes(self, note_id_list: List[str] = config.TIEBA_SPECIFIED_ID_LIST):
"""
Concurrently obtain the specified post list and save the data
Get the information and comments of the specified post
Args:
note_id_list:
Returns:
"""
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_note_detail(
note_id=post_item.get("note_id"),
semaphore=semaphore
)
for post_item in note_list
]
note_details = await asyncio.gather(*task_list)
for note_detail in note_details:
if note_detail:
await tieba_store.update_tieba_note(note_detail)
async def get_specified_notes(self):
"""Get the information and comments of the specified post"""
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_note_detail(note_id=note_id, semaphore=semaphore) for note_id in config.TIEBA_SPECIFIED_ID_LIST
self.get_note_detail_async_task(note_id=note_id, semaphore=semaphore) for note_id in note_id_list
]
note_details = await asyncio.gather(*task_list)
for note_detail in note_details:
@@ -135,11 +119,20 @@ class TieBaCrawler(AbstractCrawler):
await tieba_store.update_tieba_note(note_detail)
await self.batch_get_note_comments(config.TIEBA_SPECIFIED_ID_LIST)
async def get_note_detail(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
"""Get note detail"""
async def get_note_detail_async_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[TiebaNote]:
"""
Get note detail
Args:
note_id: baidu tieba note id
semaphore: asyncio semaphore
Returns:
"""
async with semaphore:
try:
note_detail: Dict = await self.tieba_client.get_note_by_id(note_id)
utils.logger.info(f"[BaiduTieBaCrawler.get_note_detail] Begin get note detail, note_id: {note_id}")
note_detail: TiebaNote = await self.tieba_client.get_note_by_id(note_id)
if not note_detail:
utils.logger.error(
f"[BaiduTieBaCrawler.get_note_detail] Get note detail error, note_id: {note_id}")
@@ -153,23 +146,38 @@ class TieBaCrawler(AbstractCrawler):
f"[BaiduTieBaCrawler.get_note_detail] have not fund note detail note_id:{note_id}, err: {ex}")
return None
async def batch_get_note_comments(self, note_list: List[str]):
"""Batch get note comments"""
async def batch_get_note_comments(self, note_id_list: List[str]):
"""
Batch get note comments
Args:
note_id_list:
Returns:
"""
if not config.ENABLE_GET_COMMENTS:
utils.logger.info(f"[BaiduTieBaCrawler.batch_get_note_comments] Crawling comment mode is not enabled")
return
utils.logger.info(
f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_list}")
f"[BaiduTieBaCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_id_list}")
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list: List[Task] = []
for note_id in note_list:
task = asyncio.create_task(self.get_comments(note_id, semaphore), name=note_id)
for note_id in note_id_list:
task = asyncio.create_task(self.get_comments_async_task(note_id, semaphore), name=note_id)
task_list.append(task)
await asyncio.gather(*task_list)
async def get_comments(self, note_id: str, semaphore: asyncio.Semaphore):
"""Get note comments with keyword filtering and quantity limitation"""
async def get_comments_async_task(self, note_id: str, semaphore: asyncio.Semaphore):
"""
Get comments async task
Args:
note_id:
semaphore:
Returns:
"""
async with semaphore:
utils.logger.info(f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_id}")
await self.tieba_client.get_note_all_comments(
@@ -178,23 +186,6 @@ class TieBaCrawler(AbstractCrawler):
callback=tieba_store.batch_update_tieba_note_comments
)
async def create_tieba_client(self, ip_pool: ProxyIpPool) -> BaiduTieBaClient:
"""
Create tieba client
Args:
ip_pool:
Returns:
"""
"""Create tieba client"""
utils.logger.info("[BaiduTieBaCrawler.create_tieba_client] Begin create baidutieba API client ...")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
tieba_client_obj = BaiduTieBaClient(
ip_pool=ip_pool,
)
return tieba_client_obj
async def launch_browser(
self,
chromium: BrowserType,
@@ -202,7 +193,17 @@ class TieBaCrawler(AbstractCrawler):
user_agent: Optional[str],
headless: bool = True
) -> BrowserContext:
"""Launch browser and create browser context"""
"""
Launch browser and create browser
Args:
chromium:
playwright_proxy:
user_agent:
headless:
Returns:
"""
utils.logger.info("[BaiduTieBaCrawler.launch_browser] Begin create browser context ...")
if config.SAVE_LOGIN_STATE:
# feat issue #14
@@ -227,6 +228,10 @@ class TieBaCrawler(AbstractCrawler):
return browser_context
async def close(self):
"""Close browser context"""
"""
Close browser context
Returns:
"""
await self.browser_context.close()
utils.logger.info("[BaiduTieBaCrawler.close] Browser context closed ...")