From 7edf3bcc1521f0acb8edb8f5f7f361faee7de919 Mon Sep 17 00:00:00 2001 From: mirza-samad-ahmed-baig Date: Fri, 4 Jul 2025 21:04:10 +0500 Subject: [PATCH] refactor(bilibili): process creator videos in batches --- media_platform/bilibili/core.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index b0f6dbf..9d1b74e 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -290,16 +290,14 @@ class BilibiliCrawler(AbstractCrawler): """ ps = 30 pn = 1 - video_bvids_list = [] while True: result = await self.bili_client.get_creator_videos(creator_id, pn, ps) - for video in result["list"]["vlist"]: - video_bvids_list.append(video["bvid"]) - if (int(result["page"]["count"]) <= pn * ps): + video_bvids_list = [video["bvid"] for video in result["list"]["vlist"]] + await self.get_specified_videos(video_bvids_list) + if int(result["page"]["count"]) <= pn * ps: break await asyncio.sleep(random.random()) pn += 1 - await self.get_specified_videos(video_bvids_list) async def get_specified_videos(self, bvids_list: List[str]): """