jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-05-31 23:17:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: xhs 非代理模式下增加随机等待间隔, db存储模式下增加存储xsec_token字段

Browse Source
This commit is contained in:
liudongkai
2024-12-05 21:10:31 +08:00
parent 9c7e1d499b
commit 33e7ef016d
4 changed files with 29 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
config/base_config.py
View File

@@ -21,10 +21,15 @@ PUBLISH_TIME_TYPE = 0
CRAWLER_TYPE = (
"search" # 爬取类型，search(关键词搜索) | detail(帖子详情)| creator(创作者主页数据)
)
# 自定义User Agent（暂时仅对XHS有效）
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0'
# 是否开启 IP 代理
ENABLE_IP_PROXY = False
# 未启用代理时的最大爬取间隔，单位秒（暂时仅对XHS有效）
CRAWLER_MAX_SLEEP_SEC = 2
# 代理IP池数量
IP_PROXY_POOL_COUNT = 2