jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-08 10:57:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

finish_all_for_expand_bili

Browse Source
This commit is contained in:
Bowenwin
2025-05-22 22:26:30 +08:00
parent 59619fff0a
commit 66843f216a
12 changed files with 51 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
store/bilibili/bilibili_store_impl.py
View File

@@ -38,15 +38,13 @@ def calculate_number_of_files(file_store_path: str) -> int:
if not os.path.exists(file_store_path):
return 1
try:
return max([int(file_name.split("_")[0]) for file_name in os.listdir(file_store_path)]) + 1
return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
except ValueError:
return 1
class BiliCsvStoreImplement(AbstractStore):
csv_store_path: str = "data/bilibili"
file_count: int = calculate_number_of_files(csv_store_path)
file_count:int=calculate_number_of_files(csv_store_path)
def make_save_file_name(self, store_type: str) -> str:
"""
make save file name by store type
@@ -196,7 +194,7 @@ class BiliDbStoreImplement(AbstractStore):
creator["add_ts"] = utils.get_current_timestamp()
await add_new_creator(creator)
else:
await update_creator_by_creator_id(creator_id, creator_item=creator)
await update_creator_by_creator_id(creator_id,creator_item=creator)
async def store_contact(self, contact_item: Dict):
"""
@@ -249,10 +247,11 @@ class BiliJsonStoreImplement(AbstractStore):
json_store_path: str = "data/bilibili/json"
words_store_path: str = "data/bilibili/words"
lock = asyncio.Lock()
file_count: int = calculate_number_of_files(json_store_path)
file_count:int=calculate_number_of_files(json_store_path)
WordCloud = words.AsyncWordCloudGenerator()
def make_save_file_name(self, store_type: str) -> (str, str):
def make_save_file_name(self, store_type: str) -> (str,str):
"""
make save file name by store type
Args:
@@ -279,7 +278,7 @@ class BiliJsonStoreImplement(AbstractStore):
"""
pathlib.Path(self.json_store_path).mkdir(parents=True, exist_ok=True)
pathlib.Path(self.words_store_path).mkdir(parents=True, exist_ok=True)
save_file_name, words_file_name_prefix = self.make_save_file_name(store_type=store_type)
save_file_name,words_file_name_prefix = self.make_save_file_name(store_type=store_type)
save_data = []
async with self.lock:

1
store/bilibili/bilibili_store_sql.py
View File

@@ -66,6 +66,7 @@ async def update_content_by_content_id(content_id: str, content_item: Dict) -> i
return effect_row
async def query_comment_by_comment_id(comment_id: str) -> Dict:
"""
查询一条评论内容

2
store/douyin/douyin_store_impl.py
View File

@@ -238,7 +238,7 @@ class DouyinJsonStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict):
"""
comment JSON storage implementatio
comment JSON storage implementation
Args:
comment_item:

2
store/kuaishou/kuaishou_store_impl.py
View File

@@ -215,7 +215,7 @@ class KuaishouJsonStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict):
"""
comment JSON storage implementatio
comment JSON storage implementation
Args:
comment_item:

2
store/tieba/tieba_store_impl.py
View File

@@ -235,7 +235,7 @@ class TieBaJsonStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict):
"""
comment JSON storage implementatio
comment JSON storage implementation
Args:
comment_item:

2
store/weibo/weibo_store_impl.py
View File

@@ -241,7 +241,7 @@ class WeiboJsonStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict):
"""
comment JSON storage implementatio
comment JSON storage implementation
Args:
comment_item:

2
store/xhs/xhs_store_impl.py
View File

@@ -236,7 +236,7 @@ class XhsJsonStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict):
"""
comment JSON storage implementatio
comment JSON storage implementation
Args:
comment_item:

2
store/zhihu/zhihu_store_impl.py
View File

@@ -235,7 +235,7 @@ class ZhihuJsonStoreImplement(AbstractStore):
async def store_comment(self, comment_item: Dict):
"""
comment JSON storage implementatio
comment JSON storage implementation
Args:
comment_item: