jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-06 09:57:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: issue #14

Browse Source 
refactor: 优化小红书crawler流程代码
This commit is contained in:
Relakkes
2023-07-15 17:11:53 +08:00
parent e5f4ecd8ec
commit dad8d56ab5
6 changed files with 138 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
config/base_config.py
View File

@@ -1,8 +1,7 @@
# Desc: base config
PLATFORM = "xhs"
KEYWORDS = "健身,旅游"
LOGIN_TYPE = "qrcode" # qrcode or phone or cookies
# If it's on the Xiaohongshu platform, only the web_session cookie will be kept.
# xhs cookie format -> web_session=040069b2acxxxxxxxxxxxxxxxxxxxx;
COOKIES = ""
# redis config
@@ -17,3 +16,12 @@ RETRY_INTERVAL = 60 * 30 # 30 minutes
# playwright headless
HEADLESS = True
# save login state
SAVE_LOGIN_STATE = True
# save user data dir
USER_DATA_DIR = "%s_user_data_dir" # %s will be replaced by platform name
# max page num
MAX_PAGE_NUM = 20