From 0d0af57a01e342a88bb9f261519c92d7cf7aa1e9 Mon Sep 17 00:00:00 2001 From: persist-1 Date: Wed, 10 Sep 2025 23:47:05 +0800 Subject: [PATCH] =?UTF-8?q?fix(store):=20=E4=BF=AE=E5=A4=8D'crawler=5Ftype?= =?UTF-8?q?=5Fvar'=E7=9A=84=E4=B8=8D=E5=BD=93=E4=BD=BF=E7=94=A8=E5=AF=BC?= =?UTF-8?q?=E8=87=B4csv/json=E4=BF=9D=E5=AD=98=E6=96=87=E4=BB=B6=E5=90=8D?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- store/kuaishou/_store_impl.py | 4 ++-- store/tieba/_store_impl.py | 4 ++-- store/weibo/_store_impl.py | 4 ++-- store/xhs/_store_impl.py | 6 +++--- store/zhihu/_store_impl.py | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/store/kuaishou/_store_impl.py b/store/kuaishou/_store_impl.py index 1cb1d80..4292cb0 100644 --- a/store/kuaishou/_store_impl.py +++ b/store/kuaishou/_store_impl.py @@ -50,7 +50,7 @@ def calculate_number_of_files(file_store_path: str) -> int: class KuaishouCsvStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="kuaishou", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="kuaishou", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ @@ -127,7 +127,7 @@ class KuaishouDbStoreImplement(AbstractStore): class KuaishouJsonStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="kuaishou", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="kuaishou", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ diff --git a/store/tieba/_store_impl.py b/store/tieba/_store_impl.py index 80464dd..723d1f4 100644 --- a/store/tieba/_store_impl.py +++ b/store/tieba/_store_impl.py @@ -51,7 +51,7 @@ def calculate_number_of_files(file_store_path: str) -> int: class TieBaCsvStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="tieba", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="tieba", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ @@ -149,7 +149,7 @@ class TieBaDbStoreImplement(AbstractStore): class TieBaJsonStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="tieba", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="tieba", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ diff --git a/store/weibo/_store_impl.py b/store/weibo/_store_impl.py index b27317f..f7503db 100644 --- a/store/weibo/_store_impl.py +++ b/store/weibo/_store_impl.py @@ -51,7 +51,7 @@ def calculate_number_of_files(file_store_path: str) -> int: class WeiboCsvStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="weibo", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="weibo", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ @@ -171,7 +171,7 @@ class WeiboDbStoreImplement(AbstractStore): class WeiboJsonStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="weibo", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="weibo", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ diff --git a/store/xhs/_store_impl.py b/store/xhs/_store_impl.py index 2dbfcdd..1bffddd 100644 --- a/store/xhs/_store_impl.py +++ b/store/xhs/_store_impl.py @@ -17,12 +17,12 @@ from database.models import XhsNote, XhsNoteComment, XhsCreator from tools.async_file_writer import AsyncFileWriter from tools.time_util import get_current_timestamp - +from var import crawler_type_var class XhsCsvStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="xhs", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="xhs", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ @@ -51,7 +51,7 @@ class XhsCsvStoreImplement(AbstractStore): class XhsJsonStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="xhs", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="xhs", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ diff --git a/store/zhihu/_store_impl.py b/store/zhihu/_store_impl.py index d2d6c4e..ac4dc1b 100644 --- a/store/zhihu/_store_impl.py +++ b/store/zhihu/_store_impl.py @@ -50,7 +50,7 @@ def calculate_number_of_files(file_store_path: str) -> int: class ZhihuCsvStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="zhihu", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="zhihu", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """ @@ -148,7 +148,7 @@ class ZhihuDbStoreImplement(AbstractStore): class ZhihuJsonStoreImplement(AbstractStore): def __init__(self, **kwargs): super().__init__(**kwargs) - self.writer = AsyncFileWriter(platform="zhihu", crawler_type=kwargs.get("crawler_type")) + self.writer = AsyncFileWriter(platform="zhihu", crawler_type=crawler_type_var.get()) async def store_content(self, content_item: Dict): """