mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-08 10:57:26 +08:00
新增存储到mongoDB
This commit is contained in:
@@ -28,6 +28,7 @@ from database.models import DouyinAweme, DouyinAwemeComment, DyCreator
|
||||
from tools import utils, words
|
||||
from tools.async_file_writer import AsyncFileWriter
|
||||
from var import crawler_type_var
|
||||
from store.mongodb_store_base import MongoDBStoreBase
|
||||
|
||||
|
||||
class DouyinCsvStoreImplement(AbstractStore):
|
||||
@@ -195,4 +196,62 @@ class DouyinJsonStoreImplement(AbstractStore):
|
||||
|
||||
|
||||
class DouyinSqliteStoreImplement(DouyinDbStoreImplement):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
class DouyinMongoStoreImplement(AbstractStore):
|
||||
"""抖音MongoDB存储实现"""
|
||||
|
||||
def __init__(self):
|
||||
self.mongo_store = MongoDBStoreBase(collection_prefix="douyin")
|
||||
|
||||
async def store_content(self, content_item: Dict):
|
||||
"""
|
||||
存储视频内容到MongoDB
|
||||
Args:
|
||||
content_item: 视频内容数据
|
||||
"""
|
||||
aweme_id = content_item.get("aweme_id")
|
||||
if not aweme_id:
|
||||
return
|
||||
|
||||
await self.mongo_store.save_or_update(
|
||||
collection_suffix="contents",
|
||||
query={"aweme_id": aweme_id},
|
||||
data=content_item
|
||||
)
|
||||
utils.logger.info(f"[DouyinMongoStoreImplement.store_content] Saved aweme {aweme_id} to MongoDB")
|
||||
|
||||
async def store_comment(self, comment_item: Dict):
|
||||
"""
|
||||
存储评论到MongoDB
|
||||
Args:
|
||||
comment_item: 评论数据
|
||||
"""
|
||||
comment_id = comment_item.get("comment_id")
|
||||
if not comment_id:
|
||||
return
|
||||
|
||||
await self.mongo_store.save_or_update(
|
||||
collection_suffix="comments",
|
||||
query={"comment_id": comment_id},
|
||||
data=comment_item
|
||||
)
|
||||
utils.logger.info(f"[DouyinMongoStoreImplement.store_comment] Saved comment {comment_id} to MongoDB")
|
||||
|
||||
async def store_creator(self, creator_item: Dict):
|
||||
"""
|
||||
存储创作者信息到MongoDB
|
||||
Args:
|
||||
creator_item: 创作者数据
|
||||
"""
|
||||
user_id = creator_item.get("user_id")
|
||||
if not user_id:
|
||||
return
|
||||
|
||||
await self.mongo_store.save_or_update(
|
||||
collection_suffix="creators",
|
||||
query={"user_id": user_id},
|
||||
data=creator_item
|
||||
)
|
||||
utils.logger.info(f"[DouyinMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
|
||||
Reference in New Issue
Block a user