From 076dcba978b102bb12ff69ba226d1d39158481e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Wed, 1 Jul 2026 23:32:10 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20WebUI=20=E6=BA=90?= =?UTF-8?q?=E7=A0=81=E7=BC=BA=E5=A4=B1=E3=80=81=E7=8E=AF=E5=A2=83=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E8=B7=AF=E5=BE=84=E5=8F=8A=E6=96=87=E6=A1=A3=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 调整 .gitignore，避免误忽略 webui/src/lib/ 和 webui/src/components/env/ - 补齐 WebUI 缺失的 lib 工具函数、API 封装及环境检测组件 - 修复 /api/env/check 使用相对路径导致启动目录不同时检测失败的问题 - 更新中/英/西三语 README 的 WebUI 开发调试与生产构建说明 --- .gitignore | 6 +- README.md | 32 +++- README_en.md | 30 ++- README_es.md | 32 +++- api/main.py | 9 +- webui/src/components/env/EnvironmentCheck.tsx | 177 ++++++++++++++++++ webui/src/lib/api.ts | 104 ++++++++++ webui/src/lib/urlParser.ts | 141 ++++++++++++++ webui/src/lib/utils.ts | 25 +++ 9 files changed, 528 insertions(+), 28 deletions(-) create mode 100644 webui/src/components/env/EnvironmentCheck.tsx create mode 100644 webui/src/lib/api.ts create mode 100644 webui/src/lib/urlParser.ts create mode 100644 webui/src/lib/utils.ts diff --git a/.gitignore b/.gitignore index dc49fb1..87bf8fc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +/lib/ lib64/ parts/ sdist/ @@ -122,9 +122,9 @@ celerybeat.pid # Environments .env .venv -env/ +/env/ venv/ -ENV/ +/ENV/ env.bak/ venv.bak/ diff --git a/README.md b/README.md index b1a0aa1..0c11c45 100644 --- a/README.md +++ b/README.md @@ -175,9 +175,27 @@ uv run main.py --help MediaCrawler 提供了基于 Web 的可视化操作界面，无需命令行也能轻松使用爬虫功能。 -#### 构建前端资源（首次使用需要） +#### 开发调试（推荐） -WebUI 前端源码位于 `webui/` 目录，需要先构建静态资源后才能通过 API 服务器访问。 +开发时需要同时启动后端 API 服务和前端 Vite 开发服务器： + +```shell +# 终端 1：启动 API 服务器（默认端口 8080） +uv run uvicorn api.main:app --port 8080 --reload + +# 终端 2：启动前端开发服务器 +cd webui +npm install +npm run dev # 默认在 5173 端口启动，并代理 /api 到 8080 +``` + +启动成功后，访问 `http://localhost:5173/` 即可打开 WebUI 界面。 + +> 首次打开会进行环境检测（调用 `/api/env/check`），请确保后端服务已启动。如果检测失败，可点击「跳过检测」临时跳过。 + +#### 构建生产资源 + +如果希望通过 API 服务器直接提供 WebUI 静态资源，需要先构建前端： ```shell cd webui @@ -185,19 +203,13 @@ npm install npm run build # 产物输出到 api/webui/ ``` -> 开发调试可改用 `npm run dev`，会在 5173 端口启动 Vite 开发服务器并自动代理 `/api` 到 8080。 - -#### 启动 WebUI 服务 +构建完成后，只需启动 API 服务器： ```shell -# 启动 API 服务器（默认端口 8080） uv run uvicorn api.main:app --port 8080 --reload - -# 或者使用模块方式启动 -uv run python -m api.main ``` -启动成功后，访问 `http://localhost:8080` 即可打开 WebUI 界面。 +然后访问 `http://localhost:8080` 即可。 #### WebUI 功能特性 diff --git a/README_en.md b/README_en.md index cbd4f6e..36a2d48 100644 --- a/README_en.md +++ b/README_en.md @@ -157,9 +157,27 @@ uv run main.py --help MediaCrawler provides a web-based visual operation interface, allowing you to easily use crawler features without command line. -#### Build Frontend Assets (required for first-time use) +#### Development (recommended) -The WebUI frontend source lives in the `webui/` directory. You need to build the static assets before the API server can serve them. +For development, you need to start both the backend API service and the frontend Vite dev server: + +```shell +# Terminal 1: start API server (default port 8080) +uv run uvicorn api.main:app --port 8080 --reload + +# Terminal 2: start frontend dev server +cd webui +npm install +npm run dev # starts on port 5173 by default and proxies /api to 8080 +``` + +After successful startup, visit `http://localhost:5173/` to open the WebUI interface. + +> On first launch, an environment check is performed (calls `/api/env/check`), so make sure the backend service is running. If the check fails, you can click "Skip Check" to bypass it temporarily. + +#### Build for Production + +If you want the API server to serve the WebUI static assets directly, build the frontend first: ```shell cd webui @@ -167,16 +185,10 @@ npm install npm run build # outputs to api/webui/ ``` -> For development, run `npm run dev` instead — it starts the Vite dev server on port 5173 and proxies `/api` to 8080. - -#### Start WebUI Service +Then start only the API server: ```shell -# Start API server (default port 8080) uv run uvicorn api.main:app --port 8080 --reload - -# Or start using module method -uv run python -m api.main ``` After successful startup, visit `http://localhost:8080` to open the WebUI interface. diff --git a/README_es.md b/README_es.md index 8bb3f3c..66974c8 100644 --- a/README_es.md +++ b/README_es.md @@ -157,14 +157,38 @@ uv run main.py --help MediaCrawler proporciona una interfaz de operación visual basada en web, permitiéndole usar fácilmente las funciones del rastreador sin línea de comandos. -#### Iniciar Servicio WebUI +#### Desarrollo (recomendado) + +Para el desarrollo, debe iniciar tanto el servicio API backend como el servidor de desarrollo Vite frontend: ```shell -# Iniciar servidor API (puerto predeterminado 8080) +# Terminal 1: iniciar servidor API (puerto predeterminado 8080) uv run uvicorn api.main:app --port 8080 --reload -# O iniciar usando método de módulo -uv run python -m api.main +# Terminal 2: iniciar servidor de desarrollo frontend +cd webui +npm install +npm run dev # se inicia en el puerto 5173 por defecto y redirige /api a 8080 +``` + +Después de iniciar exitosamente, visite `http://localhost:5173/` para abrir la interfaz WebUI. + +> En el primer inicio, se realiza una verificación de entorno (llama a `/api/env/check`), así que asegúrese de que el servicio backend esté en ejecución. Si la verificación falla, puede hacer clic en "Omitir verificación" para omitirla temporalmente. + +#### Construir para Producción + +Si desea que el servidor API sirva directamente los recursos estáticos de WebUI, construya el frontend primero: + +```shell +cd webui +npm install +npm run build # salida en api/webui/ +``` + +Luego inicie solo el servidor API: + +```shell +uv run uvicorn api.main:app --port 8080 --reload ``` Después de iniciar exitosamente, visite `http://localhost:8080` para abrir la interfaz WebUI. diff --git a/api/main.py b/api/main.py index 82f1664..af49b15 100644 --- a/api/main.py +++ b/api/main.py @@ -25,6 +25,7 @@ import asyncio import os import sys import subprocess +from pathlib import Path import uvicorn from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -33,6 +34,9 @@ from fastapi.responses import FileResponse from .routers import crawler_router, data_router, websocket_router +# Project root directory (used for running subprocesses like uv run main.py) +PROJECT_ROOT = Path(__file__).parent.parent + app = FastAPI( title="MediaCrawler WebUI API", description="API for controlling MediaCrawler from WebUI", @@ -86,6 +90,7 @@ async def check_environment(): """Check if MediaCrawler environment is configured correctly""" try: # Run uv run main.py --help command to check environment + # Use PROJECT_ROOT so it works regardless of where uvicorn was started if sys.platform == "win32": loop = asyncio.get_running_loop() process = await loop.run_in_executor( @@ -94,7 +99,7 @@ async def check_environment(): ["uv", "run", "main.py", "--help"], capture_output=True, timeout=30.0, - cwd="." + cwd=str(PROJECT_ROOT) ) ) stdout, stderr = process.stdout, process.stderr # bytes @@ -103,7 +108,7 @@ async def check_environment(): "uv", "run", "main.py", "--help", stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd="." # Project root directory + cwd=str(PROJECT_ROOT) # Project root directory ) stdout, stderr = await asyncio.wait_for( process.communicate(), diff --git a/webui/src/components/env/EnvironmentCheck.tsx b/webui/src/components/env/EnvironmentCheck.tsx new file mode 100644 index 0000000..148c72d --- /dev/null +++ b/webui/src/components/env/EnvironmentCheck.tsx @@ -0,0 +1,177 @@ +import { useState, useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import { CheckCircle, XCircle, Loader2, RefreshCw, AlertTriangle } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { envApi, EnvCheckResult } from '@/lib/api' + +const ENV_CHECK_KEY = 'mediacrawler_env_checked' + +interface EnvironmentCheckProps { + onCheckComplete: (success: boolean) => void +} + +// 检查是否已经通过环境检测 +export function isEnvChecked(): boolean { + return localStorage.getItem(ENV_CHECK_KEY) === 'true' +} + +// 清除环境检测状态 +export function clearEnvCheck(): void { + localStorage.removeItem(ENV_CHECK_KEY) +} + +export function EnvironmentCheck({ onCheckComplete }: EnvironmentCheckProps) { + const { t } = useTranslation('env') + const [status, setStatus] = useState<'checking' | 'success' | 'error'>('checking') + const [result, setResult] = useState(null) + const [showDetails, setShowDetails] = useState(false) + + const checkEnvironment = async () => { + setStatus('checking') + setResult(null) + try { + const response = await envApi.check() + setResult(response.data) + if (response.data.success) { + setStatus('success') + // 存储到 localStorage + localStorage.setItem(ENV_CHECK_KEY, 'true') + // 成功后延迟关闭 + setTimeout(() => onCheckComplete(true), 1500) + } else { + setStatus('error') + } + } catch (error) { + setResult({ + success: false, + message: t('defaultError'), + error: t('defaultErrorHint') + }) + setStatus('error') + } + } + + useEffect(() => { + checkEnvironment() + }, []) + + const handleSkip = () => { + localStorage.setItem(ENV_CHECK_KEY, 'true') + onCheckComplete(false) + } + + const handleRetry = () => { + checkEnvironment() + } + + return ( +
+
+ {/* Corner decorations */} +
+
+
+
+ + {/* Header */} +
+ +

+ {t('title')} +

+
+ + {/* Status Display */} +
+
+ {status === 'checking' && ( + <> + + + {t('scanning')} + + + )} + {status === 'success' && ( + <> + + + {t('success', { message: result?.message })} + + + )} + {status === 'error' && ( + <> + + + {t('error', { message: result?.message })} + + + )} +
+ + {/* Error Details */} + {status === 'error' && result?.error && ( +
+ + {showDetails && ( + 
+                  {result.error}
+
+ )} +
+ )} +
+ + {/* Help Text */} + {status === 'error' && ( +
+

{t('requirements')}

+
    +
  1. {t('requirementsList.1')}
    2. +
  2. {t('requirementsList.2')}
    3. +
  3. {t('requirementsList.3')}
    4. +
+
+ )} + + {/* Actions */} +
+ {status === 'error' && ( + <> + + + + )} + {status === 'checking' && ( + + )} +
+
+
+ ) +} diff --git a/webui/src/lib/api.ts b/webui/src/lib/api.ts new file mode 100644 index 0000000..bb27a39 --- /dev/null +++ b/webui/src/lib/api.ts @@ -0,0 +1,104 @@ +import axios from 'axios' + +const api = axios.create({ + baseURL: '/api', + timeout: 30000, + headers: { + 'Content-Type': 'application/json', + }, +}) + +// Types +export interface CrawlerConfig { + platform: string + login_type: string + crawler_type: string + keywords: string + start_page: number + enable_comments: boolean + enable_sub_comments: boolean + save_option: string + cookies: string + headless: boolean +} + +export interface CrawlerStatus { + status: 'idle' | 'running' | 'stopping' | 'error' + platform: string | null + crawler_type: string | null + started_at: string | null + error_message: string | null +} + +export interface LogEntry { + id: number + timestamp: string + level: 'info' | 'warning' | 'error' | 'success' | 'debug' + message: string +} + +export interface DataFile { + name: string + path: string + size: number + modified_at: number + record_count: number | null + type: string +} + +export interface FilePreviewResponse { + data: Record[] + total: number + columns?: string[] +} + +export interface Platform { + value: string + label: string + icon: string +} + +export interface ConfigOption { + value: string + label: string +} + +// API functions +export const crawlerApi = { + start: (config: CrawlerConfig) => api.post('/crawler/start', config), + stop: () => api.post('/crawler/stop'), + getStatus: () => api.get('/crawler/status'), + getLogs: (limit = 100) => api.get<{ logs: LogEntry[] }>('/crawler/logs', { params: { limit } }), +} + +export const dataApi = { + getFiles: (platform?: string, fileType?: string) => + api.get<{ files: DataFile[] }>('/data/files', { params: { platform, file_type: fileType } }), + getFileContent: (path: string, limit = 100) => + api.get('/data/files/' + path, { params: { preview: true, limit } }), + getStats: () => api.get('/data/stats'), + getDownloadUrl: (path: string) => `/api/data/download/${path}`, +} + +export const configApi = { + getPlatforms: () => api.get<{ platforms: Platform[] }>('/config/platforms'), + getOptions: () => + api.get<{ + login_types: ConfigOption[] + crawler_types: ConfigOption[] + save_options: ConfigOption[] + }>('/config/options'), +} + +export interface EnvCheckResult { + success: boolean + message: string + output?: string + error?: string +} + +export const envApi = { + check: () => api.get('/env/check'), +} + +export default api diff --git a/webui/src/lib/urlParser.ts b/webui/src/lib/urlParser.ts new file mode 100644 index 0000000..69acb6e --- /dev/null +++ b/webui/src/lib/urlParser.ts @@ -0,0 +1,141 @@ +export interface ParsedId { + id: string + type: 'video' | 'creator' | 'unknown' + original: string + isValid: boolean +} + +// URL patterns for different platforms +const platformPatterns: Record = { + xhs: { + video: [ + /xiaohongshu\.com\/explore\/([a-zA-Z0-9]+)/, + /xiaohongshu\.com\/discovery\/item\/([a-zA-Z0-9]+)/, + /xhslink\.com\/([a-zA-Z0-9]+)/, + ], + creator: [ + /xiaohongshu\.com\/user\/profile\/([a-zA-Z0-9]+)/, + ], + }, + dy: { + video: [ + /douyin\.com\/video\/(\d+)/, + /v\.douyin\.com\/([a-zA-Z0-9]+)/, + /iesdouyin\.com\/share\/video\/(\d+)/, + ], + creator: [ + /douyin\.com\/user\/([a-zA-Z0-9_-]+)/, + ], + }, + bili: { + video: [ + /bilibili\.com\/video\/(BV[a-zA-Z0-9]+)/, + /bilibili\.com\/video\/(av\d+)/, + /b23\.tv\/([a-zA-Z0-9]+)/, + ], + creator: [ + /space\.bilibili\.com\/(\d+)/, + ], + }, + wb: { + video: [ + /weibo\.com\/\d+\/([a-zA-Z0-9]+)/, + /m\.weibo\.cn\/status\/(\d+)/, + ], + creator: [ + /weibo\.com\/u\/(\d+)/, + /weibo\.com\/([a-zA-Z0-9]+)$/, + ], + }, + ks: { + video: [ + /kuaishou\.com\/short-video\/([a-zA-Z0-9_-]+)/, + /v\.kuaishou\.com\/([a-zA-Z0-9]+)/, + ], + creator: [ + /kuaishou\.com\/profile\/([a-zA-Z0-9_-]+)/, + ], + }, +} + +export function parseUrl(input: string, platform: string): ParsedId { + const trimmed = input.trim() + + // If it's just an ID (no URL structure), return as unknown type + if (!trimmed.includes('/') && !trimmed.includes('.')) { + return { + id: trimmed, + type: 'unknown', + original: trimmed, + isValid: trimmed.length > 0, + } + } + + const patterns = platformPatterns[platform] + if (!patterns) { + return { + id: trimmed, + type: 'unknown', + original: trimmed, + isValid: false, + } + } + + // Try video patterns + for (const pattern of patterns.video) { + const match = trimmed.match(pattern) + if (match && match[1]) { + return { + id: match[1], + type: 'video', + original: trimmed, + isValid: true, + } + } + } + + // Try creator patterns + for (const pattern of patterns.creator) { + const match = trimmed.match(pattern) + if (match && match[1]) { + return { + id: match[1], + type: 'creator', + original: trimmed, + isValid: true, + } + } + } + + // Fallback: try to extract any ID-like segment + const urlMatch = trimmed.match(/([a-zA-Z0-9_-]{6,})/) + if (urlMatch) { + return { + id: urlMatch[1], + type: 'unknown', + original: trimmed, + isValid: false, + } + } + + return { + id: trimmed, + type: 'unknown', + original: trimmed, + isValid: false, + } +} + +export function parseMultipleUrls(input: string, platform: string): ParsedId[] { + if (!input.trim()) return [] + + const items = input + .split(/[,\n]+/) + .map(s => s.trim()) + .filter(Boolean) + + return items.map(item => parseUrl(item, platform)) +} diff --git a/webui/src/lib/utils.ts b/webui/src/lib/utils.ts new file mode 100644 index 0000000..9fbf991 --- /dev/null +++ b/webui/src/lib/utils.ts @@ -0,0 +1,25 @@ +import { type ClassValue, clsx } from 'clsx' +import { twMerge } from 'tailwind-merge' + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)) +} + +export function formatFileSize(bytes: number): string { + if (bytes === 0) return '0 B' + const k = 1024 + const sizes = ['B', 'KB', 'MB', 'GB', 'TB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i] +} + +export function formatDateTime(date: string | Date | number): string { + const d = typeof date === 'string' ? new Date(date) : typeof date === 'number' ? new Date(date * 1000) : date + return d.toLocaleString('zh-CN', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }) +}