jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-04-21 19:27:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 代码优化

Browse Source
This commit is contained in:
Relakkes
2024-01-16 00:40:07 +08:00
parent e490123fcd
commit e0f9a487e4
9 changed files with 163 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
store/weibo/weibo_store_impl.py
View File

@@ -2,6 +2,7 @@
# @Author : relakkes@gmail.com
# @Time : 2024/1/14 21:35
# @Desc : 微博存储实现类
import asyncio
import csv
import json
import os
@@ -124,6 +125,7 @@ class WeiboDbStoreImplement(AbstractStore):
class WeiboJsonStoreImplement(AbstractStore):
json_store_path: str = "data/weibo"
lock = asyncio.Lock()
def make_save_file_name(self, store_type: str) -> str:
"""
@@ -150,13 +152,15 @@ class WeiboJsonStoreImplement(AbstractStore):
save_file_name = self.make_save_file_name(store_type=store_type)
save_data = []
if os.path.exists(save_file_name):
async with aiofiles.open(save_file_name, 'r', encoding='utf-8') as file:
save_data = json.loads(await file.read())
async with self.lock:
if os.path.exists(save_file_name):
async with aiofiles.open(save_file_name, 'r', encoding='utf-8') as file:
save_data = json.loads(await file.read())
save_data.append(save_item)
async with aiofiles.open(save_file_name, 'w', encoding='utf-8') as file:
await file.write(json.dumps(save_data, ensure_ascii=False))
save_data.append(save_item)
async with aiofiles.open(save_file_name, 'w', encoding='utf-8') as file:
await file.write(json.dumps(save_data, ensure_ascii=False))
async def store_content(self, content_item: Dict):
"""