mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-08 19:07:33 +08:00
refactor(config): move platform-specific configs to separate files
- Remove platform-specific configurations from base_config.py - Create separate config files for each platform in their respective directories - Update import statements in core files to use new platform-specific config modules - Clean up unused and deprecated configuration options
This commit is contained in:
24
media_platform/xhs/config.py
Normal file
24
media_platform/xhs/config.py
Normal file
@@ -0,0 +1,24 @@
|
||||
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
from config import *
|
||||
|
||||
# 小红书平台配置
|
||||
SORT_TYPE = "popularity_descending"
|
||||
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0"
|
||||
CRAWLER_MAX_SLEEP_SEC = 2
|
||||
XHS_SPECIFIED_NOTE_URL_LIST = [
|
||||
"https://www.xiaohongshu.com/explore/66fad51c000000001b0224b8?xsec_token=AB3rO-QopW5sgrJ41GwN01WCXh6yWPxjSoFI9D5JIMgKw=&xsec_source=pc_search"
|
||||
# ........................
|
||||
]
|
||||
XHS_CREATOR_ID_LIST = [
|
||||
"63e36c9a000000002703502b",
|
||||
# ........................
|
||||
]
|
||||
@@ -19,7 +19,7 @@ from typing import Dict, List, Optional, Tuple
|
||||
from playwright.async_api import BrowserContext, BrowserType, Page, Playwright, async_playwright
|
||||
from tenacity import RetryError
|
||||
|
||||
import config
|
||||
from . import config
|
||||
from base.base_crawler import AbstractCrawler
|
||||
from config import CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES
|
||||
from model.m_xiaohongshu import NoteUrlInfo
|
||||
|
||||
Reference in New Issue
Block a user