From cad9fc7af8affddfa2c556c2d6f1ead8600bcfb6 Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Sat, 12 Jul 2025 14:50:59 +0800 Subject: [PATCH 1/8] feat: Add daily limit for video/post crawling in Bilibili and base config --- config/base_config.py | 3 +++ media_platform/bilibili/core.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/config/base_config.py b/config/base_config.py index 1411f8c..815efd3 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -86,6 +86,9 @@ START_PAGE = 1 # 爬取视频/帖子的数量控制 CRAWLER_MAX_NOTES_COUNT = 200 +# 每天爬取视频/帖子的数量控制 +MAX_NOTES_PER_DAY = 20 + # 并发爬虫数量控制 MAX_CONCURRENCY_NUM = 1 diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 9d1b74e..f113c63 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -196,10 +196,14 @@ class BilibiliCrawler(AbstractCrawler): # 按照每一天进行爬取的时间戳参数 pubtime_begin_s, pubtime_end_s = await self.get_pubtime_datetime(start=day.strftime('%Y-%m-%d'), end=day.strftime('%Y-%m-%d')) page = 1 + notes_count_this_day = 0 #!该段 while 语句在发生异常时（通常情况下为当天数据为空时）会自动跳转到下一天，以实现最大程度爬取该关键词下当天的所有视频 #!除了仅保留现在原有的 try, except Exception 语句外，不要再添加其他的异常处理！！！否则将使该段代码失效，使其仅能爬取当天一天数据而无法跳转到下一天 #!除非将该段代码的逻辑进行重构以实现相同的功能，否则不要进行修改！！！ while (page - start_page + 1) * bili_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: + if notes_count_this_day >= config.MAX_NOTES_PER_DAY: + utils.logger.info(f"[BilibiliCrawler.search] Reached the maximum number of notes for today {day.ctime()}.") + break #! Catch any error if response return nothing, go to next day try: #! Don't skip any page, to make sure gather all video in one day @@ -225,6 +229,7 @@ class BilibiliCrawler(AbstractCrawler): video_items = await asyncio.gather(*task_list) for video_item in video_items: if video_item: + notes_count_this_day += 1 video_id_list.append(video_item.get("View").get("aid")) await bilibili_store.update_bilibili_video(video_item) await bilibili_store.update_up_info(video_item) From d0d72939264566849543ef8bc88b0587b5f256ba Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Sun, 13 Jul 2025 06:07:13 +0800 Subject: [PATCH 2/8] feat(bilibili): Add flexible search modes and fix limit logic Refactors the Bilibili keyword search functionality to provide more flexible crawling strategies and corrects a flaw in how crawl limits were applied. Previously, the `ALL_DAY` boolean flag offered a rigid choice for time-based searching and contained a logical issue where `CRAWLER_MAX_NOTES_COUNT` was incorrectly applied on a per-day basis instead of as an overall total. This commit introduces the `BILI_SEARCH_MODE` configuration option with three distinct modes: - `normal`: The default search behavior without time constraints. - `all_in_time_range`: Maximizes data collection within a specified date range, replicating the original intent of `ALL_DAY=True`. - `daily_limit_in_time_range`: A new mode that strictly enforces both the daily `MAX_NOTES_PER_DAY` and the total `CRAWLER_MAX_NOTES_COUNT` limits across the entire date range. This change resolves the limit logic bug and gives users more precise control over the crawling process. Changes include: - Modified `config/base_config.py` to replace `ALL_DAY` with `BILI_SEARCH_MODE`. - Refactored `media_platform/bilibili/core.py` to implement the new search mode logic. --- config/base_config.py | 17 +-- media_platform/bilibili/core.py | 206 ++++++++++++++++++-------------- 2 files changed, 128 insertions(+), 95 deletions(-) diff --git a/config/base_config.py b/config/base_config.py index 815efd3..26aeaa7 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -87,7 +87,7 @@ START_PAGE = 1 CRAWLER_MAX_NOTES_COUNT = 200 # 每天爬取视频/帖子的数量控制 -MAX_NOTES_PER_DAY = 20 +MAX_NOTES_PER_DAY = 1 # 并发爬虫数量控制 MAX_CONCURRENCY_NUM = 1 @@ -99,7 +99,7 @@ ENABLE_GET_IMAGES = False ENABLE_GET_COMMENTS = True # 爬取一级评论的数量控制(单视频/帖子) -CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 10 +CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 1 # 是否开启爬二级评论模式, 默认不开启爬二级评论 # 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段 @@ -216,16 +216,17 @@ STOP_WORDS_FILE = "./docs/hit_stopwords.txt" # 中文字体文件路径 FONT_PATH = "./docs/STZHONGS.TTF" -# 爬取开始的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式，若为 None 则表示不设置时间范围，按照默认关键字最多返回 1000 条视频的结果处理 +# 爬取开始的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式 START_DAY = "2024-01-01" -# 爬取结束的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式，若为 None 则表示不设置时间范围，按照默认关键字最多返回 1000 条视频的结果处理 +# 爬取结束的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式 END_DAY = "2024-01-01" -# 是否开启按每一天进行爬取的选项，仅支持 bilibili 关键字搜索 -# 若为 False，则忽略 START_DAY 与 END_DAY 设置的值 -# 若为 True，则按照 START_DAY 至 END_DAY 按照每一天进行筛选，这样能够突破 1000 条视频的限制，最大程度爬取该关键词下的所有视频 -ALL_DAY = False +# Bilibili 搜索模式，仅在 CRAWLER_TYPE="search" 时生效 +# "normal": 不指定时间范围进行搜索，最多返回约1000条结果。 +# "all_in_time_range": 在 START_DAY 和 END_DAY 指定的时间范围内，尽可能多地爬取数据，每日上限受 MAX_NOTES_PER_DAY 影响，但总数可能超过 CRAWLER_MAX_NOTES_COUNT。 +# "daily_limit_in_time_range": 在指定时间范围内，严格遵守 MAX_NOTES_PER_DAY 的每日上限和 CRAWLER_MAX_NOTES_COUNT 的总上限。 +BILI_SEARCH_MODE = "all_in_time_range" #!!! 下面仅支持 bilibili creator搜索 # 爬取评论creator主页还是爬取creator动态和关系列表(True为前者) diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index f113c63..5fd3e9b 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -96,7 +96,14 @@ class BilibiliCrawler(AbstractCrawler): crawler_type_var.set(config.CRAWLER_TYPE) if config.CRAWLER_TYPE == "search": # Search for video and retrieve their comment information. - await self.search() + if config.BILI_SEARCH_MODE == "normal": + await self.search_by_keywords() + elif config.BILI_SEARCH_MODE == "all_in_time_range": + await self.search_by_keywords_in_time_range(daily_limit=False) + elif config.BILI_SEARCH_MODE == "daily_limit_in_time_range": + await self.search_by_keywords_in_time_range(daily_limit=True) + else: + utils.logger.warning(f"Unknown BILI_SEARCH_MODE: {config.BILI_SEARCH_MODE}") elif config.CRAWLER_TYPE == "detail": # Get the information and comments of the specified post await self.get_specified_videos(config.BILI_SPECIFIED_ID_LIST) @@ -141,106 +148,131 @@ class BilibiliCrawler(AbstractCrawler): # 将其重新转换为时间戳 return str(int(start_day.timestamp())), str(int(end_day.timestamp())) - async def search(self): + async def search_by_keywords(self): """ - search bilibili video with keywords + search bilibili video with keywords in normal mode :return: """ - utils.logger.info("[BilibiliCrawler.search] Begin search bilibli keywords") + utils.logger.info("[BilibiliCrawler.search_by_keywords] Begin search bilibli keywords") bili_limit_count = 20 # bilibili limit page fixed value if config.CRAWLER_MAX_NOTES_COUNT < bili_limit_count: config.CRAWLER_MAX_NOTES_COUNT = bili_limit_count start_page = config.START_PAGE # start page number for keyword in config.KEYWORDS.split(","): source_keyword_var.set(keyword) - utils.logger.info(f"[BilibiliCrawler.search] Current search keyword: {keyword}") - # 每个关键词最多返回 1000 条数据 - if not config.ALL_DAY: - page = 1 - while (page - start_page + 1) * bili_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: - if page < start_page: - utils.logger.info(f"[BilibiliCrawler.search] Skip page: {page}") - page += 1 - continue - - utils.logger.info(f"[BilibiliCrawler.search] search bilibili keyword: {keyword}, page: {page}") - video_id_list: List[str] = [] - videos_res = await self.bili_client.search_video_by_keyword( - keyword=keyword, - page=page, - page_size=bili_limit_count, - order=SearchOrderType.DEFAULT, - pubtime_begin_s=0, # 作品发布日期起始时间戳 - pubtime_end_s=0 # 作品发布日期结束日期时间戳 - ) - video_list: List[Dict] = videos_res.get("result") - - semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) - task_list = [] - try: - task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list] - except Exception as e: - utils.logger.warning(f"[BilibiliCrawler.search] error in the task list. The video for this page will not be included. {e}") - video_items = await asyncio.gather(*task_list) - for video_item in video_items: - if video_item: - video_id_list.append(video_item.get("View").get("aid")) - await bilibili_store.update_bilibili_video(video_item) - await bilibili_store.update_up_info(video_item) - await self.get_bilibili_video(video_item, semaphore) + utils.logger.info(f"[BilibiliCrawler.search_by_keywords] Current search keyword: {keyword}") + page = 1 + while (page - start_page + 1) * bili_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: + if page < start_page: + utils.logger.info(f"[BilibiliCrawler.search_by_keywords] Skip page: {page}") page += 1 - await self.batch_get_video_comments(video_id_list) - # 按照 START_DAY 至 END_DAY 按照每一天进行筛选，这样能够突破 1000 条视频的限制，最大程度爬取该关键词下每一天的所有视频 - else: - for day in pd.date_range(start=config.START_DAY, end=config.END_DAY, freq='D'): - # 按照每一天进行爬取的时间戳参数 - pubtime_begin_s, pubtime_end_s = await self.get_pubtime_datetime(start=day.strftime('%Y-%m-%d'), end=day.strftime('%Y-%m-%d')) - page = 1 - notes_count_this_day = 0 - #!该段 while 语句在发生异常时（通常情况下为当天数据为空时）会自动跳转到下一天，以实现最大程度爬取该关键词下当天的所有视频 - #!除了仅保留现在原有的 try, except Exception 语句外，不要再添加其他的异常处理！！！否则将使该段代码失效，使其仅能爬取当天一天数据而无法跳转到下一天 - #!除非将该段代码的逻辑进行重构以实现相同的功能，否则不要进行修改！！！ - while (page - start_page + 1) * bili_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: - if notes_count_this_day >= config.MAX_NOTES_PER_DAY: - utils.logger.info(f"[BilibiliCrawler.search] Reached the maximum number of notes for today {day.ctime()}.") - break - #! Catch any error if response return nothing, go to next day - try: - #! Don't skip any page, to make sure gather all video in one day - # if page < start_page: - # utils.logger.info(f"[BilibiliCrawler.search] Skip page: {page}") - # page += 1 - # continue + continue - utils.logger.info(f"[BilibiliCrawler.search] search bilibili keyword: {keyword}, date: {day.ctime()}, page: {page}") - video_id_list: List[str] = [] - videos_res = await self.bili_client.search_video_by_keyword( - keyword=keyword, - page=page, - page_size=bili_limit_count, - order=SearchOrderType.DEFAULT, - pubtime_begin_s=pubtime_begin_s, # 作品发布日期起始时间戳 - pubtime_end_s=pubtime_end_s # 作品发布日期结束日期时间戳 - ) - video_list: List[Dict] = videos_res.get("result") + utils.logger.info(f"[BilibiliCrawler.search_by_keywords] search bilibili keyword: {keyword}, page: {page}") + video_id_list: List[str] = [] + videos_res = await self.bili_client.search_video_by_keyword( + keyword=keyword, + page=page, + page_size=bili_limit_count, + order=SearchOrderType.DEFAULT, + pubtime_begin_s=0, # 作品发布日期起始时间戳 + pubtime_end_s=0 # 作品发布日期结束日期时间戳 + ) + video_list: List[Dict] = videos_res.get("result") - semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) - task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list] - video_items = await asyncio.gather(*task_list) - for video_item in video_items: - if video_item: - notes_count_this_day += 1 - video_id_list.append(video_item.get("View").get("aid")) - await bilibili_store.update_bilibili_video(video_item) - await bilibili_store.update_up_info(video_item) - await self.get_bilibili_video(video_item, semaphore) - page += 1 - await self.batch_get_video_comments(video_id_list) - # go to next day - except Exception as e: - print(e) + if not video_list: + utils.logger.info(f"[BilibiliCrawler.search_by_keywords] No more videos for '{keyword}', moving to next keyword.") + break + + semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) + task_list = [] + try: + task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list] + except Exception as e: + utils.logger.warning(f"[BilibiliCrawler.search_by_keywords] error in the task list. The video for this page will not be included. {e}") + video_items = await asyncio.gather(*task_list) + for video_item in video_items: + if video_item: + video_id_list.append(video_item.get("View").get("aid")) + await bilibili_store.update_bilibili_video(video_item) + await bilibili_store.update_up_info(video_item) + await self.get_bilibili_video(video_item, semaphore) + page += 1 + await self.batch_get_video_comments(video_id_list) + + async def search_by_keywords_in_time_range(self, daily_limit: bool): + """ + Search bilibili video with keywords in a given time range. + :param daily_limit: if True, strictly limit the number of notes per day and total. + """ + utils.logger.info(f"[BilibiliCrawler.search_by_keywords_in_time_range] Begin search with daily_limit={daily_limit}") + bili_limit_count = 20 + start_page = config.START_PAGE + + for keyword in config.KEYWORDS.split(","): + source_keyword_var.set(keyword) + utils.logger.info(f"[BilibiliCrawler.search_by_keywords_in_time_range] Current search keyword: {keyword}") + total_notes_crawled = 0 + + for day in pd.date_range(start=config.START_DAY, end=config.END_DAY, freq='D'): + if daily_limit and total_notes_crawled >= config.CRAWLER_MAX_NOTES_COUNT: + utils.logger.info(f"[BilibiliCrawler.search] Reached CRAWLER_MAX_NOTES_COUNT limit for keyword '{keyword}', skipping remaining days.") + break + + pubtime_begin_s, pubtime_end_s = await self.get_pubtime_datetime(start=day.strftime('%Y-%m-%d'), end=day.strftime('%Y-%m-%d')) + page = 1 + notes_count_this_day = 0 + + while True: + if notes_count_this_day >= config.MAX_NOTES_PER_DAY: + utils.logger.info(f"[BilibiliCrawler.search] Reached MAX_NOTES_PER_DAY limit for {day.ctime()}.") + break + if daily_limit and total_notes_crawled >= config.CRAWLER_MAX_NOTES_COUNT: + utils.logger.info(f"[BilibiliCrawler.search] Reached CRAWLER_MAX_NOTES_COUNT limit for keyword '{keyword}'.") + break + if not daily_limit and (page - start_page + 1) * bili_limit_count > config.CRAWLER_MAX_NOTES_COUNT: + # For non-daily-limit mode, we still respect the total count in a loose way per day. + break + + try: + utils.logger.info(f"[BilibiliCrawler.search] search bilibili keyword: {keyword}, date: {day.ctime()}, page: {page}") + video_id_list: List[str] = [] + videos_res = await self.bili_client.search_video_by_keyword( + keyword=keyword, + page=page, + page_size=bili_limit_count, + order=SearchOrderType.DEFAULT, + pubtime_begin_s=pubtime_begin_s, + pubtime_end_s=pubtime_end_s + ) + video_list: List[Dict] = videos_res.get("result") + + if not video_list: + utils.logger.info(f"[BilibiliCrawler.search] No more videos for '{keyword}' on {day.ctime()}, moving to next day.") break + semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) + task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list] + video_items = await asyncio.gather(*task_list) + + for video_item in video_items: + if video_item: + if daily_limit and total_notes_crawled >= config.CRAWLER_MAX_NOTES_COUNT: + break + notes_count_this_day += 1 + total_notes_crawled += 1 + video_id_list.append(video_item.get("View").get("aid")) + await bilibili_store.update_bilibili_video(video_item) + await bilibili_store.update_up_info(video_item) + await self.get_bilibili_video(video_item, semaphore) + + page += 1 + await self.batch_get_video_comments(video_id_list) + + except Exception as e: + utils.logger.error(f"[BilibiliCrawler.search] Error searching on {day.ctime()}: {e}") + break + async def batch_get_video_comments(self, video_id_list: List[str]): """ batch get video comments From e91ec750bbd1c30b565e0141c86965e691145333 Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Sun, 13 Jul 2025 10:42:15 +0800 Subject: [PATCH 3/8] feat: Enhance Bilibili crawler with retry logic and robustness This commit introduces several improvements to enhance the stability and functionality of the Bilibili crawler. - **Add Retry Logic:** Implement a retry mechanism with exponential backoff when fetching video comments. This makes the crawler more resilient to transient network issues or API errors. - **Improve Error Handling:** Add a `try...except` block to handle potential `JSONDecodeError` in the Bilibili client, preventing crashes when the API returns an invalid response. - **Ensure Clean Shutdown:** Refactor `main.py` to use a `try...finally` block, guaranteeing that the crawler and database connections are properly closed on exit, error, or `KeyboardInterrupt`. - **Update Default Config:** Adjust default configuration values to increase concurrency, enable word cloud generation by default, and refine the Bilibili search mode for more practical usage. --- config/base_config.py | 10 +++---- main.py | 28 ++++++++++-------- media_platform/bilibili/client.py | 34 ++++++++++++++++++++-- media_platform/bilibili/core.py | 47 ++++++++++++++++++++----------- 4 files changed, 83 insertions(+), 36 deletions(-) diff --git a/config/base_config.py b/config/base_config.py index 26aeaa7..c3acde8 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -90,7 +90,7 @@ CRAWLER_MAX_NOTES_COUNT = 200 MAX_NOTES_PER_DAY = 1 # 并发爬虫数量控制 -MAX_CONCURRENCY_NUM = 1 +MAX_CONCURRENCY_NUM = 5 # 是否开启爬图片模式, 默认不开启爬图片 ENABLE_GET_IMAGES = False @@ -99,7 +99,7 @@ ENABLE_GET_IMAGES = False ENABLE_GET_COMMENTS = True # 爬取一级评论的数量控制(单视频/帖子) -CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 1 +CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 6 # 是否开启爬二级评论模式, 默认不开启爬二级评论 # 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段 @@ -202,7 +202,7 @@ ZHIHU_SPECIFIED_ID_LIST = [ # 词云相关 # 是否开启生成评论词云图 -ENABLE_GET_WORDCLOUD = False +ENABLE_GET_WORDCLOUD = True # 自定义词语及其分组 # 添加规则：xx:yy 其中xx为自定义添加的词组，yy为将xx该词组分到的组名。 CUSTOM_WORDS = { @@ -220,13 +220,13 @@ FONT_PATH = "./docs/STZHONGS.TTF" START_DAY = "2024-01-01" # 爬取结束的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式 -END_DAY = "2024-01-01" +END_DAY = "2025-07-12" # Bilibili 搜索模式，仅在 CRAWLER_TYPE="search" 时生效 # "normal": 不指定时间范围进行搜索，最多返回约1000条结果。 # "all_in_time_range": 在 START_DAY 和 END_DAY 指定的时间范围内，尽可能多地爬取数据，每日上限受 MAX_NOTES_PER_DAY 影响，但总数可能超过 CRAWLER_MAX_NOTES_COUNT。 # "daily_limit_in_time_range": 在指定时间范围内，严格遵守 MAX_NOTES_PER_DAY 的每日上限和 CRAWLER_MAX_NOTES_COUNT 的总上限。 -BILI_SEARCH_MODE = "all_in_time_range" +BILI_SEARCH_MODE = "daily_limit_in_time_range" #!!! 下面仅支持 bilibili creator搜索 # 爬取评论creator主页还是爬取creator动态和关系列表(True为前者) diff --git a/main.py b/main.py index 7292701..1dc5f71 100644 --- a/main.py +++ b/main.py @@ -45,25 +45,29 @@ class CrawlerFactory: return crawler_class() async def main(): + # Init crawler + crawler: Optional[AbstractCrawler] = None + try: + # parse cmd + await cmd_arg.parse_cmd() - # parse cmd - await cmd_arg.parse_cmd() + # init db + if config.SAVE_DATA_OPTION == "db": + await db.init_db() - # init db - if config.SAVE_DATA_OPTION == "db": - await db.init_db() + crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM) + await crawler.start() + finally: + if crawler: + await crawler.close() + if config.SAVE_DATA_OPTION == "db": + await db.close() - crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM) - await crawler.start() - - if config.SAVE_DATA_OPTION == "db": - await db.close() - - if __name__ == '__main__': try: # asyncio.run(main()) asyncio.get_event_loop().run_until_complete(main()) except KeyboardInterrupt: + print("\n[main] Caught keyboard interrupt, exiting.") sys.exit() diff --git a/media_platform/bilibili/client.py b/media_platform/bilibili/client.py index 32af357..9893f31 100644 --- a/media_platform/bilibili/client.py +++ b/media_platform/bilibili/client.py @@ -53,7 +53,11 @@ class BilibiliClient(AbstractApiClient): method, url, timeout=self.timeout, **kwargs ) - data: Dict = response.json() + try: + data: Dict = response.json() + except json.JSONDecodeError: + utils.logger.error(f"[BilibiliClient.request] Failed to decode JSON from response. status_code: {response.status_code}, response_text: {response.text}") + raise DataFetchError(f"Failed to decode JSON, content: {response.text}") if data.get("code") != 0: raise DataFetchError(data.get("message", "unkonw error")) else: @@ -235,13 +239,37 @@ class BilibiliClient(AbstractApiClient): :return: """ - result = [] is_end = False next_page = 0 + max_retries = 3 while not is_end and len(result) < max_count: - comments_res = await self.get_video_comments(video_id, CommentOrderType.DEFAULT, next_page) + comments_res = None + for attempt in range(max_retries): + try: + comments_res = await self.get_video_comments(video_id, CommentOrderType.DEFAULT, next_page) + break # Success + except DataFetchError as e: + if attempt < max_retries - 1: + delay = 5 * (2 ** attempt) + random.uniform(0, 1) + utils.logger.warning( + f"[BilibiliClient.get_video_all_comments] Retrying video_id {video_id} in {delay:.2f}s... (Attempt {attempt + 1}/{max_retries})" + ) + await asyncio.sleep(delay) + else: + utils.logger.error( + f"[BilibiliClient.get_video_all_comments] Max retries reached for video_id: {video_id}. Skipping comments. Error: {e}" + ) + is_end = True + break + if not comments_res: + break + cursor_info: Dict = comments_res.get("cursor") + if not cursor_info: + utils.logger.warning(f"[BilibiliClient.get_video_all_comments] Could not find 'cursor' in response for video_id: {video_id}. Skipping.") + break + comment_list: List[Dict] = comments_res.get("replies", []) is_end = cursor_info.get("is_end") next_page = cursor_info.get("next") diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 5fd3e9b..a8c89cb 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -23,6 +23,7 @@ from datetime import datetime, timedelta import pandas as pd from playwright.async_api import (BrowserContext, BrowserType, Page, Playwright, async_playwright) +from playwright._impl._errors import TargetClosedError import config from base.base_crawler import AbstractCrawler @@ -95,15 +96,7 @@ class BilibiliCrawler(AbstractCrawler): crawler_type_var.set(config.CRAWLER_TYPE) if config.CRAWLER_TYPE == "search": - # Search for video and retrieve their comment information. - if config.BILI_SEARCH_MODE == "normal": - await self.search_by_keywords() - elif config.BILI_SEARCH_MODE == "all_in_time_range": - await self.search_by_keywords_in_time_range(daily_limit=False) - elif config.BILI_SEARCH_MODE == "daily_limit_in_time_range": - await self.search_by_keywords_in_time_range(daily_limit=True) - else: - utils.logger.warning(f"Unknown BILI_SEARCH_MODE: {config.BILI_SEARCH_MODE}") + await self.search() elif config.CRAWLER_TYPE == "detail": # Get the information and comments of the specified post await self.get_specified_videos(config.BILI_SPECIFIED_ID_LIST) @@ -118,6 +111,20 @@ class BilibiliCrawler(AbstractCrawler): utils.logger.info( "[BilibiliCrawler.start] Bilibili Crawler finished ...") + async def search(self): + """ + search bilibili video + """ + # Search for video and retrieve their comment information. + if config.BILI_SEARCH_MODE == "normal": + await self.search_by_keywords() + elif config.BILI_SEARCH_MODE == "all_in_time_range": + await self.search_by_keywords_in_time_range(daily_limit=False) + elif config.BILI_SEARCH_MODE == "daily_limit_in_time_range": + await self.search_by_keywords_in_time_range(daily_limit=True) + else: + utils.logger.warning(f"Unknown BILI_SEARCH_MODE: {config.BILI_SEARCH_MODE}") + @staticmethod async def get_pubtime_datetime(start: str = config.START_DAY, end: str = config.END_DAY) -> Tuple[str, str]: """ @@ -259,6 +266,8 @@ class BilibiliCrawler(AbstractCrawler): if video_item: if daily_limit and total_notes_crawled >= config.CRAWLER_MAX_NOTES_COUNT: break + if notes_count_this_day >= config.MAX_NOTES_PER_DAY: + break notes_count_this_day += 1 total_notes_crawled += 1 video_id_list.append(video_item.get("View").get("aid")) @@ -305,6 +314,7 @@ class BilibiliCrawler(AbstractCrawler): try: utils.logger.info( f"[BilibiliCrawler.get_comments] begin get video_id: {video_id} comments ...") + await asyncio.sleep(random.uniform(0.5, 1.5)) await self.bili_client.get_video_all_comments( video_id=video_id, crawl_interval=random.random(), @@ -509,13 +519,18 @@ class BilibiliCrawler(AbstractCrawler): async def close(self): """Close browser context""" - # 如果使用CDP模式，需要特殊处理 - if self.cdp_manager: - await self.cdp_manager.cleanup() - self.cdp_manager = None - else: - await self.browser_context.close() - utils.logger.info("[BilibiliCrawler.close] Browser context closed ...") + try: + # 如果使用CDP模式，需要特殊处理 + if self.cdp_manager: + await self.cdp_manager.cleanup() + self.cdp_manager = None + elif self.browser_context: + await self.browser_context.close() + utils.logger.info("[BilibiliCrawler.close] Browser context closed ...") + except TargetClosedError: + utils.logger.warning("[BilibiliCrawler.close] Browser context was already closed.") + except Exception as e: + utils.logger.error(f"[BilibiliCrawler.close] An error occurred during close: {e}") async def get_bilibili_video(self, video_item: Dict, semaphore: asyncio.Semaphore): """ From 4d743f6c17054585ff32b2f1987ea6120d8a5f4b Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Mon, 14 Jul 2025 08:00:48 +0800 Subject: [PATCH 4/8] debug & resume default configuration --- config/base_config.py | 10 +++++----- media_platform/bilibili/core.py | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/config/base_config.py b/config/base_config.py index c3acde8..d92648f 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -90,7 +90,7 @@ CRAWLER_MAX_NOTES_COUNT = 200 MAX_NOTES_PER_DAY = 1 # 并发爬虫数量控制 -MAX_CONCURRENCY_NUM = 5 +MAX_CONCURRENCY_NUM = 1 # 是否开启爬图片模式, 默认不开启爬图片 ENABLE_GET_IMAGES = False @@ -99,7 +99,7 @@ ENABLE_GET_IMAGES = False ENABLE_GET_COMMENTS = True # 爬取一级评论的数量控制(单视频/帖子) -CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 6 +CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 10 # 是否开启爬二级评论模式, 默认不开启爬二级评论 # 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段 @@ -202,7 +202,7 @@ ZHIHU_SPECIFIED_ID_LIST = [ # 词云相关 # 是否开启生成评论词云图 -ENABLE_GET_WORDCLOUD = True +ENABLE_GET_WORDCLOUD = False # 自定义词语及其分组 # 添加规则：xx:yy 其中xx为自定义添加的词组，yy为将xx该词组分到的组名。 CUSTOM_WORDS = { @@ -220,13 +220,13 @@ FONT_PATH = "./docs/STZHONGS.TTF" START_DAY = "2024-01-01" # 爬取结束的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式 -END_DAY = "2025-07-12" +END_DAY = "2024-01-01" # Bilibili 搜索模式，仅在 CRAWLER_TYPE="search" 时生效 # "normal": 不指定时间范围进行搜索，最多返回约1000条结果。 # "all_in_time_range": 在 START_DAY 和 END_DAY 指定的时间范围内，尽可能多地爬取数据，每日上限受 MAX_NOTES_PER_DAY 影响，但总数可能超过 CRAWLER_MAX_NOTES_COUNT。 # "daily_limit_in_time_range": 在指定时间范围内，严格遵守 MAX_NOTES_PER_DAY 的每日上限和 CRAWLER_MAX_NOTES_COUNT 的总上限。 -BILI_SEARCH_MODE = "daily_limit_in_time_range" +BILI_SEARCH_MODE = "normal" #!!! 下面仅支持 bilibili creator搜索 # 爬取评论creator主页还是爬取creator动态和关系列表(True为前者) diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index a8c89cb..6e76da4 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -329,6 +329,8 @@ class BilibiliCrawler(AbstractCrawler): except Exception as e: utils.logger.error( f"[BilibiliCrawler.get_comments] may be been blocked, err:{e}") + # Propagate the exception to be caught by the main loop + raise async def get_creator_videos(self, creator_id: int): """ From 9fb396c7d18e391347fff0e90e0616a05d7e2c30 Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Thu, 17 Jul 2025 06:40:56 +0800 Subject: [PATCH 5/8] fix(media_platform): handle edge cases and improve error handling for Bilibili client and crawler - BilibiliClient: - Improve wbi_img_urls handling for better compatibility - Add error handling for missing or invalid 'is_end' and 'next' in comment cursor - BilibiliCrawler: - Fix daily limit logic for keyword-based searches - Improve logging and break conditions for max notes count limits - Ensure proper tracking of total notes crawled for each keyword --- media_platform/bilibili/client.py | 23 +++++++++++++++++++---- media_platform/bilibili/core.py | 19 ++++++++++++------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/media_platform/bilibili/client.py b/media_platform/bilibili/client.py index 9893f31..12fb1a6 100644 --- a/media_platform/bilibili/client.py +++ b/media_platform/bilibili/client.py @@ -15,6 +15,7 @@ # @Desc : bilibili 请求客户端 import asyncio import json +import random from typing import Any, Callable, Dict, List, Optional, Tuple, Union from urllib.parse import urlencode @@ -82,8 +83,12 @@ class BilibiliClient(AbstractApiClient): :return: """ local_storage = await self.playwright_page.evaluate("() => window.localStorage") - wbi_img_urls = local_storage.get("wbi_img_urls", "") or local_storage.get( - "wbi_img_url") + "-" + local_storage.get("wbi_sub_url") + wbi_img_urls = local_storage.get("wbi_img_urls", "") + if not wbi_img_urls: + img_url_from_storage = local_storage.get("wbi_img_url") + sub_url_from_storage = local_storage.get("wbi_sub_url") + if img_url_from_storage and sub_url_from_storage: + wbi_img_urls = f"{img_url_from_storage}-{sub_url_from_storage}" if wbi_img_urls and "-" in wbi_img_urls: img_url, sub_url = wbi_img_urls.split("-") else: @@ -271,8 +276,18 @@ class BilibiliClient(AbstractApiClient): break comment_list: List[Dict] = comments_res.get("replies", []) - is_end = cursor_info.get("is_end") - next_page = cursor_info.get("next") + + # 检查 is_end 和 next 是否存在 + if "is_end" not in cursor_info or "next" not in cursor_info: + utils.logger.warning(f"[BilibiliClient.get_video_all_comments] 'is_end' or 'next' not in cursor for video_id: {video_id}. Assuming end of comments.") + is_end = True + else: + is_end = cursor_info.get("is_end") + next_page = cursor_info.get("next") + + if not isinstance(is_end, bool): + utils.logger.warning(f"[BilibiliClient.get_video_all_comments] 'is_end' is not a boolean for video_id: {video_id}. Assuming end of comments.") + is_end = True if is_fetch_sub_comments: for comment in comment_list: comment_id = comment['rpid'] diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 6e76da4..16706c7 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -219,10 +219,14 @@ class BilibiliCrawler(AbstractCrawler): for keyword in config.KEYWORDS.split(","): source_keyword_var.set(keyword) utils.logger.info(f"[BilibiliCrawler.search_by_keywords_in_time_range] Current search keyword: {keyword}") - total_notes_crawled = 0 + total_notes_crawled_for_keyword = 0 for day in pd.date_range(start=config.START_DAY, end=config.END_DAY, freq='D'): - if daily_limit and total_notes_crawled >= config.CRAWLER_MAX_NOTES_COUNT: + if daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT: + utils.logger.info(f"[BilibiliCrawler.search] Reached CRAWLER_MAX_NOTES_COUNT limit for keyword '{keyword}', skipping remaining days.") + break + + if not daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT: utils.logger.info(f"[BilibiliCrawler.search] Reached CRAWLER_MAX_NOTES_COUNT limit for keyword '{keyword}', skipping remaining days.") break @@ -234,11 +238,10 @@ class BilibiliCrawler(AbstractCrawler): if notes_count_this_day >= config.MAX_NOTES_PER_DAY: utils.logger.info(f"[BilibiliCrawler.search] Reached MAX_NOTES_PER_DAY limit for {day.ctime()}.") break - if daily_limit and total_notes_crawled >= config.CRAWLER_MAX_NOTES_COUNT: + if daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT: utils.logger.info(f"[BilibiliCrawler.search] Reached CRAWLER_MAX_NOTES_COUNT limit for keyword '{keyword}'.") break - if not daily_limit and (page - start_page + 1) * bili_limit_count > config.CRAWLER_MAX_NOTES_COUNT: - # For non-daily-limit mode, we still respect the total count in a loose way per day. + if not daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT: break try: @@ -264,12 +267,14 @@ class BilibiliCrawler(AbstractCrawler): for video_item in video_items: if video_item: - if daily_limit and total_notes_crawled >= config.CRAWLER_MAX_NOTES_COUNT: + if daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT: + break + if not daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT: break if notes_count_this_day >= config.MAX_NOTES_PER_DAY: break notes_count_this_day += 1 - total_notes_crawled += 1 + total_notes_crawled_for_keyword += 1 video_id_list.append(video_item.get("View").get("aid")) await bilibili_store.update_bilibili_video(video_item) await bilibili_store.update_up_info(video_item) From 1dc8c1789f01fc98c5eedbec775fb752aed1704e Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Thu, 17 Jul 2025 07:51:27 +0800 Subject: [PATCH 6/8] docs(config): update Bilibili search mode options - Clarify the three search mode options for Bilibili - Add note about setting MAX_NOTES_PER_DAY in bilibili config --- config/base_config.py | 7 ++++--- media_platform/bilibili/config.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 media_platform/bilibili/config.py diff --git a/config/base_config.py b/config/base_config.py index e451304..12a7abe 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -223,9 +223,10 @@ START_DAY = "2024-01-01" END_DAY = "2024-01-01" # Bilibili 搜索模式，仅在 CRAWLER_TYPE="search" 时生效 -# "normal": 不指定时间范围进行搜索，最多返回约1000条结果。 -# "all_in_time_range": 在 START_DAY 和 END_DAY 指定的时间范围内，尽可能多地爬取数据，每日上限受 MAX_NOTES_PER_DAY 影响，但总数可能超过 CRAWLER_MAX_NOTES_COUNT。 -# "daily_limit_in_time_range": 在指定时间范围内，严格遵守 MAX_NOTES_PER_DAY 的每日上限和 CRAWLER_MAX_NOTES_COUNT 的总上限。 +# 1. "normal": 不指定时间范围进行搜索，最多返回约1000条结果。 +# 2. "all_in_time_range": 在 START_DAY 和 END_DAY 指定的时间范围内，尽可能多地爬取数据，每日上限受 MAX_NOTES_PER_DAY 影响，但总数可能超过 CRAWLER_MAX_NOTES_COUNT。 +# 3. "daily_limit_in_time_range": 在指定时间范围内，严格遵守 MAX_NOTES_PER_DAY 的每日上限和 CRAWLER_MAX_NOTES_COUNT 的总上限。 +# 三个选项中，2. 和 3. 需在 media_platform\bilibili\config.py 中设定 MAX_NOTES_PER_DAY BILI_SEARCH_MODE = "normal" #!!! 下面仅支持 bilibili creator搜索 diff --git a/media_platform/bilibili/config.py b/media_platform/bilibili/config.py new file mode 100644 index 0000000..6f080cf --- /dev/null +++ b/media_platform/bilibili/config.py @@ -0,0 +1,13 @@ +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + + +# 每天爬取视频/帖子的数量控制 +MAX_NOTES_PER_DAY = 1 \ No newline at end of file From 5daae04c7d9c17741d196fc6ca02d92a94e36735 Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Thu, 17 Jul 2025 16:54:17 +0800 Subject: [PATCH 7/8] fix: conflict resolution errors --- main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index f88dfd6..dd23cf1 100644 --- a/main.py +++ b/main.py @@ -51,19 +51,19 @@ async def main(): # parse cmd await cmd_arg.parse_cmd() - # parse cmd - await cmd_arg.parse_cmd() + # init db + if config.SAVE_DATA_OPTION in ["db", "sqlite"]: + await db.init_db() - # init db - if config.SAVE_DATA_OPTION in ["db", "sqlite"]: - await db.init_db() + crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM) + await crawler.start() - crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM) - await crawler.start() - - if config.SAVE_DATA_OPTION in ["db", "sqlite"]: - await db.close() + finally: + if crawler: + await crawler.close() + if config.SAVE_DATA_OPTION in ["db", "sqlite"]: + await db.close() if __name__ == '__main__': try: From b913db64bb2d2d6d76b85ce76fe5e9c1c7c9e819 Mon Sep 17 00:00:00 2001 From: gaoxiaobei <99178334+gaoxiaobei@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:27:37 +0800 Subject: [PATCH 8/8] refactor(config): move platform-specific configs to separate files - Remove platform-specific configurations from base_config.py - Create separate config files for each platform in their respective directories - Update import statements in core files to use new platform-specific config modules - Clean up unused and deprecated configuration options --- config/__init__.py | 2 +- config/base_config.py | 138 +----------------------------- media_platform/bilibili/config.py | 23 ++++- media_platform/bilibili/core.py | 2 +- media_platform/douyin/config.py | 23 +++++ media_platform/douyin/core.py | 2 +- media_platform/kuaishou/config.py | 18 ++++ media_platform/kuaishou/core.py | 2 +- media_platform/tieba/config.py | 21 +++++ media_platform/tieba/core.py | 2 +- media_platform/weibo/config.py | 22 +++++ media_platform/weibo/core.py | 2 +- media_platform/xhs/config.py | 24 ++++++ media_platform/xhs/core.py | 2 +- media_platform/zhihu/config.py | 22 +++++ media_platform/zhihu/core.py | 2 +- 16 files changed, 161 insertions(+), 146 deletions(-) create mode 100644 media_platform/douyin/config.py create mode 100644 media_platform/kuaishou/config.py create mode 100644 media_platform/tieba/config.py create mode 100644 media_platform/weibo/config.py create mode 100644 media_platform/xhs/config.py create mode 100644 media_platform/zhihu/config.py diff --git a/config/__init__.py b/config/__init__.py index ca94b32..1c1e97c 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -10,4 +10,4 @@ from .base_config import * -from .db_config import * +from .db_config import * \ No newline at end of file diff --git a/config/base_config.py b/config/base_config.py index 12a7abe..b8d535d 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -10,28 +10,16 @@ # 基础配置 -PLATFORM = "xhs" +PLATFORM = "xhs" # 平台，xhs | dy | ks | bili | wb | tieba | zhihu KEYWORDS = "编程副业,编程兼职" # 关键词搜索配置，以英文逗号分隔 LOGIN_TYPE = "qrcode" # qrcode or phone or cookie COOKIES = "" -# 具体值参见media_platform.xxx.field下的枚举值，暂时只支持小红书 -SORT_TYPE = "popularity_descending" -# 具体值参见media_platform.xxx.field下的枚举值，暂时只支持抖音 -PUBLISH_TIME_TYPE = 0 CRAWLER_TYPE = ( "search" # 爬取类型，search(关键词搜索) | detail(帖子详情)| creator(创作者主页数据) ) -# 微博搜索类型 default (综合) | real_time (实时) | popular (热门) | video (视频) -WEIBO_SEARCH_TYPE = "popular" -# 自定义User Agent（暂时仅对XHS有效） -UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0" - # 是否开启 IP 代理 ENABLE_IP_PROXY = False -# 未启用代理时的最大爬取间隔，单位秒（暂时仅对XHS有效） -CRAWLER_MAX_SLEEP_SEC = 2 - # 代理IP池数量 IP_PROXY_POOL_COUNT = 2 @@ -86,9 +74,6 @@ START_PAGE = 1 # 爬取视频/帖子的数量控制 CRAWLER_MAX_NOTES_COUNT = 200 -# 每天爬取视频/帖子的数量控制 -MAX_NOTES_PER_DAY = 1 - # 并发爬虫数量控制 MAX_CONCURRENCY_NUM = 1 @@ -105,101 +90,6 @@ CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 10 # 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段 ENABLE_GET_SUB_COMMENTS = False -# 已废弃⚠️⚠️⚠️指定小红书需要爬虫的笔记ID列表 -# 已废弃⚠️⚠️⚠️ 指定笔记ID笔记列表会因为缺少xsec_token和xsec_source参数导致爬取失败 -# XHS_SPECIFIED_ID_LIST = [ -# "66fad51c000000001b0224b8", -# # ........................ -# ] - -# 指定小红书需要爬虫的笔记URL列表, 目前要携带xsec_token和xsec_source参数 -XHS_SPECIFIED_NOTE_URL_LIST = [ - "https://www.xiaohongshu.com/explore/66fad51c000000001b0224b8?xsec_token=AB3rO-QopW5sgrJ41GwN01WCXh6yWPxjSoFI9D5JIMgKw=&xsec_source=pc_search" - # ........................ -] - -# 指定抖音需要爬取的ID列表 -DY_SPECIFIED_ID_LIST = [ - "7280854932641664319", - "7202432992642387233", - # ........................ -] - -# 指定快手平台需要爬取的ID列表 -KS_SPECIFIED_ID_LIST = ["3xf8enb8dbj6uig", "3x6zz972bchmvqe"] - -# 指定B站平台需要爬取的视频bvid列表 -BILI_SPECIFIED_ID_LIST = [ - "BV1d54y1g7db", - "BV1Sz4y1U77N", - "BV14Q4y1n7jz", - # ........................ -] - -# 指定微博平台需要爬取的帖子列表 -WEIBO_SPECIFIED_ID_LIST = [ - "4982041758140155", - # ........................ -] - -# 指定weibo创作者ID列表 -WEIBO_CREATOR_ID_LIST = [ - "5533390220", - # ........................ -] - -# 指定贴吧需要爬取的帖子列表 -TIEBA_SPECIFIED_ID_LIST = [] - -# 指定贴吧名称列表，爬取该贴吧下的帖子 -TIEBA_NAME_LIST = [ - # "盗墓笔记" -] - -# 指定贴吧创作者URL列表 -TIEBA_CREATOR_URL_LIST = [ - "https://tieba.baidu.com/home/main/?id=tb.1.7f139e2e.6CyEwxu3VJruH_-QqpCi6g&fr=frs", - # ........................ -] - -# 指定小红书创作者ID列表 -XHS_CREATOR_ID_LIST = [ - "63e36c9a000000002703502b", - # ........................ -] - -# 指定Dy创作者ID列表(sec_id) -DY_CREATOR_ID_LIST = [ - "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE", - # ........................ -] - -# 指定bili创作者ID列表(sec_id) -BILI_CREATOR_ID_LIST = [ - "20813884", - # ........................ -] - -# 指定快手创作者ID列表 -KS_CREATOR_ID_LIST = [ - "3x4sm73aye7jq7i", - # ........................ -] - - -# 指定知乎创作者主页url列表 -ZHIHU_CREATOR_URL_LIST = [ - "https://www.zhihu.com/people/yd1234567", - # ........................ -] - -# 指定知乎需要爬取的帖子ID列表 -ZHIHU_SPECIFIED_ID_LIST = [ - "https://www.zhihu.com/question/826896610/answer/4885821440", # 回答 - "https://zhuanlan.zhihu.com/p/673461588", # 文章 - "https://www.zhihu.com/zvideo/1539542068422144000", # 视频 -] - # 词云相关 # 是否开启生成评论词云图 ENABLE_GET_WORDCLOUD = False @@ -215,29 +105,3 @@ STOP_WORDS_FILE = "./docs/hit_stopwords.txt" # 中文字体文件路径 FONT_PATH = "./docs/STZHONGS.TTF" - -# 爬取开始的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式 -START_DAY = "2024-01-01" - -# 爬取结束的天数，仅支持 bilibili 关键字搜索，YYYY-MM-DD 格式 -END_DAY = "2024-01-01" - -# Bilibili 搜索模式，仅在 CRAWLER_TYPE="search" 时生效 -# 1. "normal": 不指定时间范围进行搜索，最多返回约1000条结果。 -# 2. "all_in_time_range": 在 START_DAY 和 END_DAY 指定的时间范围内，尽可能多地爬取数据，每日上限受 MAX_NOTES_PER_DAY 影响，但总数可能超过 CRAWLER_MAX_NOTES_COUNT。 -# 3. "daily_limit_in_time_range": 在指定时间范围内，严格遵守 MAX_NOTES_PER_DAY 的每日上限和 CRAWLER_MAX_NOTES_COUNT 的总上限。 -# 三个选项中，2. 和 3. 需在 media_platform\bilibili\config.py 中设定 MAX_NOTES_PER_DAY -BILI_SEARCH_MODE = "normal" - -#!!! 下面仅支持 bilibili creator搜索 -# 爬取评论creator主页还是爬取creator动态和关系列表(True为前者) -CREATOR_MODE = True - -# 爬取creator粉丝列表时起始爬取页数 -START_CONTACTS_PAGE = 1 - -# 爬取作者粉丝和关注列表数量控制(单作者) -CRAWLER_MAX_CONTACTS_COUNT_SINGLENOTES = 100 - -# 爬取作者动态数量控制(单作者) -CRAWLER_MAX_DYNAMICS_COUNT_SINGLENOTES = 50 diff --git a/media_platform/bilibili/config.py b/media_platform/bilibili/config.py index 6f080cf..8d48a26 100644 --- a/media_platform/bilibili/config.py +++ b/media_platform/bilibili/config.py @@ -9,5 +9,26 @@ # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 +from config import * + # 每天爬取视频/帖子的数量控制 -MAX_NOTES_PER_DAY = 1 \ No newline at end of file +MAX_NOTES_PER_DAY = 1 + +# Bilibili 平台配置 +BILI_SPECIFIED_ID_LIST = [ + "BV1d54y1g7db", + "BV1Sz4y1U77N", + "BV14Q4y1n7jz", + # ........................ +] +START_DAY = "2024-01-01" +END_DAY = "2024-01-01" +BILI_SEARCH_MODE = "normal" +CREATOR_MODE = True +START_CONTACTS_PAGE = 1 +CRAWLER_MAX_CONTACTS_COUNT_SINGLENOTES = 100 +CRAWLER_MAX_DYNAMICS_COUNT_SINGLENOTES = 50 +BILI_CREATOR_ID_LIST = [ + "20813884", + # ........................ +] diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 16706c7..674316b 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -25,7 +25,7 @@ import pandas as pd from playwright.async_api import (BrowserContext, BrowserType, Page, Playwright, async_playwright) from playwright._impl._errors import TargetClosedError -import config +from . import config from base.base_crawler import AbstractCrawler from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import bilibili as bilibili_store diff --git a/media_platform/douyin/config.py b/media_platform/douyin/config.py new file mode 100644 index 0000000..c29171f --- /dev/null +++ b/media_platform/douyin/config.py @@ -0,0 +1,23 @@ +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from config import * + +# 抖音平台配置 +PUBLISH_TIME_TYPE = 0 +DY_SPECIFIED_ID_LIST = [ + "7280854932641664319", + "7202432992642387233", + # ........................ +] +DY_CREATOR_ID_LIST = [ + "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE", + # ........................ +] diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index 05027b7..3934639 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple from playwright.async_api import (BrowserContext, BrowserType, Page, Playwright, async_playwright) -import config +from . import config from base.base_crawler import AbstractCrawler from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import douyin as douyin_store diff --git a/media_platform/kuaishou/config.py b/media_platform/kuaishou/config.py new file mode 100644 index 0000000..bbd400f --- /dev/null +++ b/media_platform/kuaishou/config.py @@ -0,0 +1,18 @@ +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from config import * + +# 快手平台配置 +KS_SPECIFIED_ID_LIST = ["3xf8enb8dbj6uig", "3x6zz972bchmvqe"] +KS_CREATOR_ID_LIST = [ + "3x4sm73aye7jq7i", + # ........................ +] diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py index f92c7f2..c3410cb 100644 --- a/media_platform/kuaishou/core.py +++ b/media_platform/kuaishou/core.py @@ -18,7 +18,7 @@ from typing import Dict, List, Optional, Tuple from playwright.async_api import BrowserContext, BrowserType, Page, Playwright, async_playwright -import config +from . import config from base.base_crawler import AbstractCrawler from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import kuaishou as kuaishou_store diff --git a/media_platform/tieba/config.py b/media_platform/tieba/config.py new file mode 100644 index 0000000..e189cd3 --- /dev/null +++ b/media_platform/tieba/config.py @@ -0,0 +1,21 @@ +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from config import * + +# 贴吧平台配置 +TIEBA_SPECIFIED_ID_LIST = [] +TIEBA_NAME_LIST = [ + # "盗墓笔记" +] +TIEBA_CREATOR_URL_LIST = [ + "https://tieba.baidu.com/home/main/?id=tb.1.7f139e2e.6CyEwxu3VJruH_-QqpCi6g&fr=frs", + # ........................ +] diff --git a/media_platform/tieba/core.py b/media_platform/tieba/core.py index 01fe21d..c4087ea 100644 --- a/media_platform/tieba/core.py +++ b/media_platform/tieba/core.py @@ -18,7 +18,7 @@ from typing import Dict, List, Optional, Tuple from playwright.async_api import (BrowserContext, BrowserType, Page, Playwright, async_playwright) -import config +from . import config from base.base_crawler import AbstractCrawler from model.m_baidu_tieba import TiebaCreator, TiebaNote from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool diff --git a/media_platform/weibo/config.py b/media_platform/weibo/config.py new file mode 100644 index 0000000..aeaabaa --- /dev/null +++ b/media_platform/weibo/config.py @@ -0,0 +1,22 @@ +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from config import * + +# 微博平台配置 +WEIBO_SEARCH_TYPE = "popular" +WEIBO_SPECIFIED_ID_LIST = [ + "4982041758140155", + # ........................ +] +WEIBO_CREATOR_ID_LIST = [ + "5533390220", + # ........................ +] diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index ccc0fb1..f31d6e4 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -24,7 +24,7 @@ from typing import Dict, List, Optional, Tuple from playwright.async_api import (BrowserContext, BrowserType, Page, Playwright, async_playwright) -import config +from . import config from base.base_crawler import AbstractCrawler from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import weibo as weibo_store diff --git a/media_platform/xhs/config.py b/media_platform/xhs/config.py new file mode 100644 index 0000000..747f067 --- /dev/null +++ b/media_platform/xhs/config.py @@ -0,0 +1,24 @@ +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from config import * + +# 小红书平台配置 +SORT_TYPE = "popularity_descending" +UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0" +CRAWLER_MAX_SLEEP_SEC = 2 +XHS_SPECIFIED_NOTE_URL_LIST = [ + "https://www.xiaohongshu.com/explore/66fad51c000000001b0224b8?xsec_token=AB3rO-QopW5sgrJ41GwN01WCXh6yWPxjSoFI9D5JIMgKw=&xsec_source=pc_search" + # ........................ +] +XHS_CREATOR_ID_LIST = [ + "63e36c9a000000002703502b", + # ........................ +] diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index df65092..67965f7 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -19,7 +19,7 @@ from typing import Dict, List, Optional, Tuple from playwright.async_api import BrowserContext, BrowserType, Page, Playwright, async_playwright from tenacity import RetryError -import config +from . import config from base.base_crawler import AbstractCrawler from config import CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES from model.m_xiaohongshu import NoteUrlInfo diff --git a/media_platform/zhihu/config.py b/media_platform/zhihu/config.py new file mode 100644 index 0000000..bbbbab7 --- /dev/null +++ b/media_platform/zhihu/config.py @@ -0,0 +1,22 @@ +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from config import * + +# 知乎平台配置 +ZHIHU_CREATOR_URL_LIST = [ + "https://www.zhihu.com/people/yd1234567", + # ........................ +] +ZHIHU_SPECIFIED_ID_LIST = [ + "https://www.zhihu.com/question/826896610/answer/4885821440", # 回答 + "https://zhuanlan.zhihu.com/p/673461588", # 文章 + "https://www.zhihu.com/zvideo/1539542068422144000", # 视频 +] diff --git a/media_platform/zhihu/core.py b/media_platform/zhihu/core.py index 9c9e532..84f4bf2 100644 --- a/media_platform/zhihu/core.py +++ b/media_platform/zhihu/core.py @@ -19,7 +19,7 @@ from typing import Dict, List, Optional, Tuple, cast from playwright.async_api import (BrowserContext, BrowserType, Page, Playwright, async_playwright) -import config +from . import config from constant import zhihu as constant from base.base_crawler import AbstractCrawler from model.m_zhihu import ZhihuContent, ZhihuCreator