jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-10 03:47:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 轻量化支持爬取小红书二级评论

Browse Source
This commit is contained in:
leantli
2024-04-12 17:32:20 +08:00
parent 81a9946afd
commit ad01dfba95
8 changed files with 52 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
store/xhs/__init__.py
View File

@@ -74,6 +74,7 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
user_info = comment_item.get("user_info", {})
comment_id = comment_item.get("id")
comment_pictures = [item.get("url_default", "") for item in comment_item.get("pictures", [])]
target_comment = comment_item.get("target_comment", {})
local_db_item = {
"comment_id": comment_id,
"create_time": comment_item.get("create_time"),
@@ -83,40 +84,13 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
"user_id": user_info.get("user_id"),
"nickname": user_info.get("nickname"),
"avatar": user_info.get("image"),
"sub_comment_count": comment_item.get("sub_comment_count"),
"sub_comment_count": comment_item.get("sub_comment_count", 0),
"pictures": ",".join(comment_pictures),
"parent_comment_id": target_comment.get("id", 0),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.xhs.update_xhs_note_comment] xhs note comment:{local_db_item}")
await XhsStoreFactory.create_store().store_comment(local_db_item)
async def batch_update_xhs_note_sub_comments(note_id: str, comments: List[Dict]):
if not comments:
return
for comment_item in comments:
await update_xhs_note_sub_comment(note_id, comment_item)
async def update_xhs_note_sub_comment(note_id: str, comment_item: Dict):
user_info = comment_item.get("user_info", {})
target_comment = comment_item.get("target_comment")
comment_id = comment_item.get("id")
comment_pictures = [item.get("url_default", "") for item in comment_item.get("pictures", [])]
local_db_item = {
"comment_id": comment_id,
"target_comment_id": target_comment.get('id'),
"create_time": comment_item.get("create_time"),
"ip_location": comment_item.get("ip_location"),
"note_id": note_id,
"content": comment_item.get("content"),
"user_id": user_info.get("user_id"),
"nickname": user_info.get("nickname"),
"avatar": user_info.get("image"),
"pictures": ",".join(comment_pictures),
"last_modify_ts": utils.get_current_timestamp(),
}
utils.logger.info(f"[store.xhs.update_xhs_note_sub_comment] xhs note comment:{local_db_item}")
await XhsStoreFactory.create_store().store_sub_comment(local_db_item)
async def save_creator(user_id: str, creator: Dict):

43
store/xhs/xhs_store_impl.py
View File

@@ -81,17 +81,6 @@ class XhsCsvStoreImplement(AbstractStore):
"""
await self.save_data_to_csv(save_item=creator, store_type="creator")
async def store_sub_comment(self, sub_comment_items: Dict):
"""
Xiaohongshu sub_comments CSV storage implementation
Args:
sub_comment_items: sub_comments item dict
Returns:
"""
await self.save_data_to_csv(save_item=sub_comment_items, store_type="sub_comments")
class XhsDbStoreImplement(AbstractStore):
@@ -154,26 +143,6 @@ class XhsDbStoreImplement(AbstractStore):
else:
await update_creator_by_user_id(user_id, creator)
async def store_sub_comment(self, sub_comment_items: Dict):
"""
Xiaohongshu content DB storage implementation
Args:
sub_comment_items: comment item dict
Returns:
"""
from .xhs_store_sql import (add_new_sub_comment,
query_sub_comment_by_comment_id,
update_sub_comment_by_comment_id)
comment_id = sub_comment_items.get("comment_id")
comment_detail: Dict = await query_sub_comment_by_comment_id(comment_id=comment_id)
if not comment_detail:
sub_comment_items["add_ts"] = utils.get_current_timestamp()
await add_new_sub_comment(sub_comment_items)
else:
await update_sub_comment_by_comment_id(comment_id, comment_item=sub_comment_items)
class XhsJsonStoreImplement(AbstractStore):
json_store_path: str = "data/xhs"
@@ -245,14 +214,4 @@ class XhsJsonStoreImplement(AbstractStore):
"""
await self.save_data_to_json(creator, "creator")
async def store_sub_comment(self, sub_comment_items: Dict):
"""
sub_comment JSON storage implementatio
Args:
sub_comment_items:
Returns:
"""
await self.save_data_to_json(sub_comment_items, "sub_comments")

46
store/xhs/xhs_store_sql.py
View File

@@ -146,49 +146,3 @@ async def update_creator_by_user_id(user_id: str, creator_item: Dict) -> int:
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
effect_row: int = await async_db_conn.update_table("xhs_creator", creator_item, "user_id", user_id)
return effect_row
async def query_sub_comment_by_comment_id(comment_id: str) -> Dict:
"""
查询一条二级评论内容
Args:
comment_id:
Returns:
"""
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
sql: str = f"select * from xhs_note_sub_comment where comment_id = '{comment_id}'"
rows: List[Dict] = await async_db_conn.query(sql)
if len(rows) > 0:
return rows[0]
return dict()
async def add_new_sub_comment(comment_item: Dict) -> int:
"""
新增一条二级评论记录
Args:
comment_item:
Returns:
"""
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
last_row_id: int = await async_db_conn.item_to_table("xhs_note_sub_comment", comment_item)
return last_row_id
async def update_sub_comment_by_comment_id(comment_id: str, comment_item: Dict) -> int:
"""
更新增一条二级评论记录
Args:
comment_id:
comment_item:
Returns:
"""
async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
effect_row: int = await async_db_conn.update_table("xhs_note_sub_comment", comment_item, "comment_id", comment_id)
return effect_row