jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 03:17:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 捕捉处理 ValueError 报错以及修改错字

Browse Source
This commit is contained in:
leantli
2024-05-08 22:26:35 +08:00
parent 57ee8dd63b
commit 43acde240b
5 changed files with 43 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
store/xhs/xhs_store_impl.py
View File

@@ -15,7 +15,8 @@ from base.base_crawler import AbstractStore
from tools import utils
from var import crawler_type_var
def calculatet_number_of_files(file_store_path: str) -> int:
def calculate_number_of_files(file_store_path: str) -> int:
"""计算数据保存文件的前部分排序数字，支持每次运行代码不写到同一个文件中
Args:
file_store_path;
@@ -24,12 +25,15 @@ def calculatet_number_of_files(file_store_path: str) -> int:
"""
if not os.path.exists(file_store_path):
return 1
return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
try:
return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
except ValueError:
return 1
class XhsCsvStoreImplement(AbstractStore):
csv_store_path: str = "data/xhs"
file_count:int=calculatet_number_of_files(csv_store_path)
file_count:int=calculate_number_of_files(csv_store_path)
def make_save_file_name(self, store_type: str) -> str:
"""
@@ -159,7 +163,7 @@ class XhsDbStoreImplement(AbstractStore):
class XhsJsonStoreImplement(AbstractStore):
json_store_path: str = "data/xhs"
lock = asyncio.Lock()
file_count:int=calculatet_number_of_files(json_store_path)
file_count:int=calculate_number_of_files(json_store_path)
def make_save_file_name(self, store_type: str) -> str:
"""