mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-07 18:37:30 +08:00
feat(api): add WebUI API server with built frontend
- Add FastAPI server with WebSocket support for real-time logs - Add crawler management API endpoints (start/stop/status) - Add data browsing API endpoints (list files, preview, download) - Include pre-built WebUI assets for serving frontend API endpoints: - POST /api/crawler/start - Start crawler task - POST /api/crawler/stop - Stop crawler task - GET /api/crawler/status - Get crawler status - WS /api/ws/logs - Real-time log streaming - GET /api/data/files - List data files - GET /api/data/stats - Get data statistics 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
程序员阿江(Relakkes)
21
api/services/__init__.py
Normal file
21
api/services/__init__.py
Normal file
@@ -0,0 +1,21 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 relakkes@gmail.com
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/services/__init__.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
from .crawler_manager import CrawlerManager, crawler_manager
|
||||
|
||||
__all__ = ["CrawlerManager", "crawler_manager"]
|
||||
281
api/services/crawler_manager.py
Normal file
281
api/services/crawler_manager.py
Normal file
@@ -0,0 +1,281 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 relakkes@gmail.com
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/services/crawler_manager.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
import signal
|
||||
import os
|
||||
from typing import Optional, List
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from ..schemas import CrawlerStartRequest, LogEntry
|
||||
|
||||
|
||||
class CrawlerManager:
|
||||
"""爬虫进程管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = asyncio.Lock()
|
||||
self.process: Optional[subprocess.Popen] = None
|
||||
self.status = "idle"
|
||||
self.started_at: Optional[datetime] = None
|
||||
self.current_config: Optional[CrawlerStartRequest] = None
|
||||
self._log_id = 0
|
||||
self._logs: List[LogEntry] = []
|
||||
self._read_task: Optional[asyncio.Task] = None
|
||||
# 项目根目录
|
||||
self._project_root = Path(__file__).parent.parent.parent
|
||||
# 日志队列 - 用于向 WebSocket 推送
|
||||
self._log_queue: Optional[asyncio.Queue] = None
|
||||
|
||||
@property
|
||||
def logs(self) -> List[LogEntry]:
|
||||
return self._logs
|
||||
|
||||
def get_log_queue(self) -> asyncio.Queue:
|
||||
"""获取或创建日志队列"""
|
||||
if self._log_queue is None:
|
||||
self._log_queue = asyncio.Queue()
|
||||
return self._log_queue
|
||||
|
||||
def _create_log_entry(self, message: str, level: str = "info") -> LogEntry:
|
||||
"""创建日志条目"""
|
||||
self._log_id += 1
|
||||
entry = LogEntry(
|
||||
id=self._log_id,
|
||||
timestamp=datetime.now().strftime("%H:%M:%S"),
|
||||
level=level,
|
||||
message=message
|
||||
)
|
||||
self._logs.append(entry)
|
||||
# 保留最近 500 条日志
|
||||
if len(self._logs) > 500:
|
||||
self._logs = self._logs[-500:]
|
||||
return entry
|
||||
|
||||
async def _push_log(self, entry: LogEntry):
|
||||
"""推送日志到队列"""
|
||||
if self._log_queue is not None:
|
||||
try:
|
||||
self._log_queue.put_nowait(entry)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
def _parse_log_level(self, line: str) -> str:
|
||||
"""解析日志级别"""
|
||||
line_upper = line.upper()
|
||||
if "ERROR" in line_upper or "FAILED" in line_upper:
|
||||
return "error"
|
||||
elif "WARNING" in line_upper or "WARN" in line_upper:
|
||||
return "warning"
|
||||
elif "SUCCESS" in line_upper or "完成" in line or "成功" in line:
|
||||
return "success"
|
||||
elif "DEBUG" in line_upper:
|
||||
return "debug"
|
||||
return "info"
|
||||
|
||||
async def start(self, config: CrawlerStartRequest) -> bool:
|
||||
"""启动爬虫进程"""
|
||||
async with self._lock:
|
||||
if self.process and self.process.poll() is None:
|
||||
return False
|
||||
|
||||
# 清空旧日志
|
||||
self._logs = []
|
||||
self._log_id = 0
|
||||
|
||||
# 清空待推送队列（不要替换对象，避免 WebSocket 广播协程持有旧队列引用）
|
||||
if self._log_queue is None:
|
||||
self._log_queue = asyncio.Queue()
|
||||
else:
|
||||
try:
|
||||
while True:
|
||||
self._log_queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
pass
|
||||
|
||||
# 构建命令行参数
|
||||
cmd = self._build_command(config)
|
||||
|
||||
# 记录启动日志
|
||||
entry = self._create_log_entry(f"Starting crawler: {' '.join(cmd)}", "info")
|
||||
await self._push_log(entry)
|
||||
|
||||
try:
|
||||
# 启动子进程
|
||||
self.process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
cwd=str(self._project_root),
|
||||
env={**os.environ, "PYTHONUNBUFFERED": "1"}
|
||||
)
|
||||
|
||||
self.status = "running"
|
||||
self.started_at = datetime.now()
|
||||
self.current_config = config
|
||||
|
||||
entry = self._create_log_entry(
|
||||
f"Crawler started on platform: {config.platform.value}, type: {config.crawler_type.value}",
|
||||
"success"
|
||||
)
|
||||
await self._push_log(entry)
|
||||
|
||||
# 启动日志读取任务
|
||||
self._read_task = asyncio.create_task(self._read_output())
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
self.status = "error"
|
||||
entry = self._create_log_entry(f"Failed to start crawler: {str(e)}", "error")
|
||||
await self._push_log(entry)
|
||||
return False
|
||||
|
||||
async def stop(self) -> bool:
|
||||
"""停止爬虫进程"""
|
||||
async with self._lock:
|
||||
if not self.process or self.process.poll() is not None:
|
||||
return False
|
||||
|
||||
self.status = "stopping"
|
||||
entry = self._create_log_entry("Sending SIGTERM to crawler process...", "warning")
|
||||
await self._push_log(entry)
|
||||
|
||||
try:
|
||||
self.process.send_signal(signal.SIGTERM)
|
||||
|
||||
# 等待优雅退出 (最多15秒)
|
||||
for _ in range(30):
|
||||
if self.process.poll() is not None:
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# 如果还没退出，强制杀死
|
||||
if self.process.poll() is None:
|
||||
entry = self._create_log_entry("Process not responding, sending SIGKILL...", "warning")
|
||||
await self._push_log(entry)
|
||||
self.process.kill()
|
||||
|
||||
entry = self._create_log_entry("Crawler process terminated", "info")
|
||||
await self._push_log(entry)
|
||||
|
||||
except Exception as e:
|
||||
entry = self._create_log_entry(f"Error stopping crawler: {str(e)}", "error")
|
||||
await self._push_log(entry)
|
||||
|
||||
self.status = "idle"
|
||||
self.current_config = None
|
||||
|
||||
# 取消日志读取任务
|
||||
if self._read_task:
|
||||
self._read_task.cancel()
|
||||
self._read_task = None
|
||||
|
||||
return True
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取当前状态"""
|
||||
return {
|
||||
"status": self.status,
|
||||
"platform": self.current_config.platform.value if self.current_config else None,
|
||||
"crawler_type": self.current_config.crawler_type.value if self.current_config else None,
|
||||
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||
"error_message": None
|
||||
}
|
||||
|
||||
def _build_command(self, config: CrawlerStartRequest) -> list:
|
||||
"""构建 main.py 命令行参数"""
|
||||
cmd = ["python", "main.py"]
|
||||
|
||||
cmd.extend(["--platform", config.platform.value])
|
||||
cmd.extend(["--lt", config.login_type.value])
|
||||
cmd.extend(["--type", config.crawler_type.value])
|
||||
cmd.extend(["--save_data_option", config.save_option.value])
|
||||
|
||||
# 根据爬虫类型传递不同的参数
|
||||
if config.crawler_type.value == "search" and config.keywords:
|
||||
cmd.extend(["--keywords", config.keywords])
|
||||
elif config.crawler_type.value == "detail" and config.specified_ids:
|
||||
cmd.extend(["--specified_id", config.specified_ids])
|
||||
elif config.crawler_type.value == "creator" and config.creator_ids:
|
||||
cmd.extend(["--creator_id", config.creator_ids])
|
||||
|
||||
if config.start_page != 1:
|
||||
cmd.extend(["--start", str(config.start_page)])
|
||||
|
||||
cmd.extend(["--get_comment", "true" if config.enable_comments else "false"])
|
||||
cmd.extend(["--get_sub_comment", "true" if config.enable_sub_comments else "false"])
|
||||
|
||||
if config.cookies:
|
||||
cmd.extend(["--cookies", config.cookies])
|
||||
|
||||
cmd.extend(["--headless", "true" if config.headless else "false"])
|
||||
|
||||
return cmd
|
||||
|
||||
async def _read_output(self):
|
||||
"""异步读取进程输出"""
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
try:
|
||||
while self.process and self.process.poll() is None:
|
||||
# 在线程池中读取一行
|
||||
line = await loop.run_in_executor(
|
||||
None, self.process.stdout.readline
|
||||
)
|
||||
if line:
|
||||
line = line.strip()
|
||||
if line:
|
||||
level = self._parse_log_level(line)
|
||||
entry = self._create_log_entry(line, level)
|
||||
await self._push_log(entry)
|
||||
|
||||
# 读取剩余输出
|
||||
if self.process and self.process.stdout:
|
||||
remaining = await loop.run_in_executor(
|
||||
None, self.process.stdout.read
|
||||
)
|
||||
if remaining:
|
||||
for line in remaining.strip().split('\n'):
|
||||
if line.strip():
|
||||
level = self._parse_log_level(line)
|
||||
entry = self._create_log_entry(line.strip(), level)
|
||||
await self._push_log(entry)
|
||||
|
||||
# 进程结束
|
||||
if self.status == "running":
|
||||
exit_code = self.process.returncode if self.process else -1
|
||||
if exit_code == 0:
|
||||
entry = self._create_log_entry("Crawler completed successfully", "success")
|
||||
else:
|
||||
entry = self._create_log_entry(f"Crawler exited with code: {exit_code}", "warning")
|
||||
await self._push_log(entry)
|
||||
self.status = "idle"
|
||||
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
entry = self._create_log_entry(f"Error reading output: {str(e)}", "error")
|
||||
await self._push_log(entry)
|
||||
|
||||
|
||||
# 全局单例
|
||||
crawler_manager = CrawlerManager()
|
||||
Reference in New Issue
Block a user