feat: 增加搜索词来源渠道

This commit is contained in:
Relakkes
2024-08-23 08:29:24 +08:00
parent d3c5111985
commit c70bd9e071
15 changed files with 37 additions and 8 deletions

View File

@@ -12,7 +12,7 @@ from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import douyin as douyin_store
from tools import utils
from var import crawler_type_var
from var import crawler_type_var, source_keyword_var
from .client import DOUYINClient
from .exception import DataFetchError
@@ -80,6 +80,7 @@ class DouYinCrawler(AbstractCrawler):
config.CRAWLER_MAX_NOTES_COUNT = dy_limit_count
start_page = config.START_PAGE # start page number
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(f"[DouYinCrawler.search] Current keyword: {keyword}")
aweme_list: List[str] = []
page = 0