jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-03-18 09:17:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 增加搜索词来源渠道

Browse Source
This commit is contained in:
Relakkes
2024-08-23 08:29:24 +08:00
parent d3c5111985
commit c70bd9e071
15 changed files with 37 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
media_platform/bilibili/core.py
View File

@@ -17,7 +17,7 @@ from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import bilibili as bilibili_store
from tools import utils
from var import crawler_type_var
from var import crawler_type_var, source_keyword_var
from .client import BilibiliClient
from .exception import DataFetchError
@@ -96,6 +96,7 @@ class BilibiliCrawler(AbstractCrawler):
config.CRAWLER_MAX_NOTES_COUNT = bili_limit_count
start_page = config.START_PAGE # start page number
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(
f"[BilibiliCrawler.search] Current search keyword: {keyword}")
page = 1

3
media_platform/douyin/core.py
View File

@@ -12,7 +12,7 @@ from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import douyin as douyin_store
from tools import utils
from var import crawler_type_var
from var import crawler_type_var, source_keyword_var
from .client import DOUYINClient
from .exception import DataFetchError
@@ -80,6 +80,7 @@ class DouYinCrawler(AbstractCrawler):
config.CRAWLER_MAX_NOTES_COUNT = dy_limit_count
start_page = config.START_PAGE # start page number
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(f"[DouYinCrawler.search] Current keyword: {keyword}")
aweme_list: List[str] = []
page = 0

3
media_platform/kuaishou/core.py
View File

@@ -13,7 +13,7 @@ from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import kuaishou as kuaishou_store
from tools import utils
from var import comment_tasks_var, crawler_type_var
from var import comment_tasks_var, crawler_type_var, source_keyword_var
from .client import KuaiShouClient
from .exception import DataFetchError
@@ -85,6 +85,7 @@ class KuaishouCrawler(AbstractCrawler):
config.CRAWLER_MAX_NOTES_COUNT = ks_limit_count
start_page = config.START_PAGE
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(f"[KuaishouCrawler.search] Current search keyword: {keyword}")
page = 1
while (page - start_page + 1) * ks_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:

3
media_platform/tieba/core.py
View File

@@ -14,7 +14,7 @@ from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import tieba as tieba_store
from tools import utils
from tools.crawler_util import format_proxy_info
from var import crawler_type_var
from var import crawler_type_var, source_keyword_var
from .client import BaiduTieBaClient
from .field import SearchNoteType, SearchSortType
@@ -74,6 +74,7 @@ class TieBaCrawler(AbstractCrawler):
config.CRAWLER_MAX_NOTES_COUNT = tieba_limit_count
start_page = config.START_PAGE
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(f"[BaiduTieBaCrawler.search] Current search keyword: {keyword}")
page = 1
while (page - start_page + 1) * tieba_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:

3
media_platform/weibo/core.py
View File

@@ -18,7 +18,7 @@ from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import weibo as weibo_store
from tools import utils
from var import crawler_type_var
from var import crawler_type_var, source_keyword_var
from .client import WeiboClient
from .exception import DataFetchError
@@ -99,6 +99,7 @@ class WeiboCrawler(AbstractCrawler):
config.CRAWLER_MAX_NOTES_COUNT = weibo_limit_count
start_page = config.START_PAGE
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(f"[WeiboCrawler.search] Current search keyword: {keyword}")
page = 1
while (page - start_page + 1) * weibo_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:

3
media_platform/xhs/core.py
View File

@@ -12,7 +12,7 @@ from base.base_crawler import AbstractCrawler
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import xhs as xhs_store
from tools import utils
from var import crawler_type_var
from var import crawler_type_var, source_keyword_var
from .client import XiaoHongShuClient
from .exception import DataFetchError
@@ -94,6 +94,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
config.CRAWLER_MAX_NOTES_COUNT = xhs_limit_count
start_page = config.START_PAGE
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(f"[XiaoHongShuCrawler.search] Current search keyword: {keyword}")
page = 1
while (page - start_page + 1) * xhs_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: