jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-04-22 03:37:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 捕捉处理 ValueError 报错以及修改错字

Browse Source
This commit is contained in:
leantli
2024-05-08 22:26:35 +08:00
parent 57ee8dd63b
commit 43acde240b
5 changed files with 43 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
store/kuaishou/kuaishou_store_impl.py
View File

@@ -14,7 +14,9 @@ import aiofiles
from base.base_crawler import AbstractStore
from tools import utils
from var import crawler_type_var
def calculatet_number_of_files(file_store_path: str) -> int:
def calculate_number_of_files(file_store_path: str) -> int:
"""计算数据保存文件的前部分排序数字，支持每次运行代码不写到同一个文件中
Args:
file_store_path;
@@ -23,12 +25,15 @@ def calculatet_number_of_files(file_store_path: str) -> int:
"""
if not os.path.exists(file_store_path):
return 1
return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
try:
return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
except ValueError:
return 1
class KuaishouCsvStoreImplement(AbstractStore):
csv_store_path: str = "data/kuaishou"
file_count:int=calculatet_number_of_files(csv_store_path)
file_count:int=calculate_number_of_files(csv_store_path)
def make_save_file_name(self, store_type: str) -> str:
"""
@@ -128,7 +133,7 @@ class KuaishouDbStoreImplement(AbstractStore):
class KuaishouJsonStoreImplement(AbstractStore):
json_store_path: str = "data/kuaishou"
lock = asyncio.Lock()
file_count:int=calculatet_number_of_files(json_store_path)
file_count:int=calculate_number_of_files(json_store_path)
def make_save_file_name(self, store_type: str) -> str: