From cae707cb2a43eb3883727d1a82f82ea60965f3ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 18 Oct 2025 07:00:21 +0800 Subject: [PATCH] feat: douyin support url link --- config/dy_config.py | 22 +++++-- media_platform/douyin/client.py | 25 ++++++++ media_platform/douyin/core.py | 45 ++++++++++++-- media_platform/douyin/help.py | 105 ++++++++++++++++++++++++++++++++ model/m_douyin.py | 29 ++++++--- 5 files changed, 208 insertions(+), 18 deletions(-) diff --git a/config/dy_config.py b/config/dy_config.py index b974dca..cd36065 100644 --- a/config/dy_config.py +++ b/config/dy_config.py @@ -11,15 +11,27 @@ # 抖音平台配置 PUBLISH_TIME_TYPE = 0 -# 指定DY视频ID列表 +# 指定DY视频URL列表 (支持多种格式) +# 支持格式: +# 1. 完整视频URL: "https://www.douyin.com/video/7525538910311632128" +# 2. 带modal_id的URL: "https://www.douyin.com/user/xxx?modal_id=7525538910311632128" +# 3. 搜索页带modal_id: "https://www.douyin.com/root/search/python?modal_id=7525538910311632128" +# 4. 短链接: "https://v.douyin.com/drIPtQ_WPWY/" +# 5. 纯视频ID: "7280854932641664319" DY_SPECIFIED_ID_LIST = [ - "7280854932641664319", - "7202432992642387233", + "https://www.douyin.com/video/7525538910311632128", + "https://v.douyin.com/drIPtQ_WPWY/", + "https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main&modal_id=7525538910311632128", + "7202432992642387233", # ........................ ] -# 指定DY用户ID列表 +# 指定DY创作者URL列表 (支持完整URL或sec_user_id) +# 支持格式: +# 1. 完整创作者主页URL: "https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main" +# 2. sec_user_id: "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE" DY_CREATOR_ID_LIST = [ - "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE", + "https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main", + "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE" # ........................ ] diff --git a/media_platform/douyin/client.py b/media_platform/douyin/client.py index 46a3e8f..5d980ec 100644 --- a/media_platform/douyin/client.py +++ b/media_platform/douyin/client.py @@ -324,3 +324,28 @@ class DouYinClient(AbstractApiClient): except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试 return None + + async def resolve_short_url(self, short_url: str) -> str: + """ + 解析抖音短链接,获取重定向后的真实URL + Args: + short_url: 短链接,如 https://v.douyin.com/iF12345ABC/ + Returns: + 重定向后的完整URL + """ + async with httpx.AsyncClient(proxy=self.proxy, follow_redirects=False) as client: + try: + utils.logger.info(f"[DouYinClient.resolve_short_url] Resolving short URL: {short_url}") + response = await client.get(short_url, timeout=10) + + # 短链接通常返回302重定向 + if response.status_code in [301, 302, 303, 307, 308]: + redirect_url = response.headers.get("Location", "") + utils.logger.info(f"[DouYinClient.resolve_short_url] Resolved to: {redirect_url}") + return redirect_url + else: + utils.logger.warning(f"[DouYinClient.resolve_short_url] Unexpected status code: {response.status_code}") + return "" + except Exception as e: + utils.logger.error(f"[DouYinClient.resolve_short_url] Failed to resolve short URL: {e}") + return "" diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index 191e6ab..92aebb3 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -33,6 +33,7 @@ from var import crawler_type_var, source_keyword_var from .client import DouYinClient from .exception import DataFetchError from .field import PublishTimeType +from .help import parse_video_info_from_url, parse_creator_info_from_url from .login import DouYinLogin @@ -154,15 +155,39 @@ class DouYinCrawler(AbstractCrawler): await self.batch_get_note_comments(aweme_list) async def get_specified_awemes(self): - """Get the information and comments of the specified post""" + """Get the information and comments of the specified post from URLs or IDs""" + utils.logger.info("[DouYinCrawler.get_specified_awemes] Parsing video URLs...") + aweme_id_list = [] + for video_url in config.DY_SPECIFIED_ID_LIST: + try: + video_info = parse_video_info_from_url(video_url) + + # 处理短链接 + if video_info.url_type == "short": + utils.logger.info(f"[DouYinCrawler.get_specified_awemes] Resolving short link: {video_url}") + resolved_url = await self.dy_client.resolve_short_url(video_url) + if resolved_url: + # 从解析后的URL中提取视频ID + video_info = parse_video_info_from_url(resolved_url) + utils.logger.info(f"[DouYinCrawler.get_specified_awemes] Short link resolved to aweme ID: {video_info.aweme_id}") + else: + utils.logger.error(f"[DouYinCrawler.get_specified_awemes] Failed to resolve short link: {video_url}") + continue + + aweme_id_list.append(video_info.aweme_id) + utils.logger.info(f"[DouYinCrawler.get_specified_awemes] Parsed aweme ID: {video_info.aweme_id} from {video_url}") + except ValueError as e: + utils.logger.error(f"[DouYinCrawler.get_specified_awemes] Failed to parse video URL: {e}") + continue + semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) - task_list = [self.get_aweme_detail(aweme_id=aweme_id, semaphore=semaphore) for aweme_id in config.DY_SPECIFIED_ID_LIST] + task_list = [self.get_aweme_detail(aweme_id=aweme_id, semaphore=semaphore) for aweme_id in aweme_id_list] aweme_details = await asyncio.gather(*task_list) for aweme_detail in aweme_details: if aweme_detail is not None: await douyin_store.update_douyin_aweme(aweme_item=aweme_detail) await self.get_aweme_media(aweme_item=aweme_detail) - await self.batch_get_note_comments(config.DY_SPECIFIED_ID_LIST) + await self.batch_get_note_comments(aweme_id_list) async def get_aweme_detail(self, aweme_id: str, semaphore: asyncio.Semaphore) -> Any: """Get note detail""" @@ -218,10 +243,20 @@ class DouYinCrawler(AbstractCrawler): async def get_creators_and_videos(self) -> None: """ - Get the information and videos of the specified creator + Get the information and videos of the specified creator from URLs or IDs """ utils.logger.info("[DouYinCrawler.get_creators_and_videos] Begin get douyin creators") - for user_id in config.DY_CREATOR_ID_LIST: + utils.logger.info("[DouYinCrawler.get_creators_and_videos] Parsing creator URLs...") + + for creator_url in config.DY_CREATOR_ID_LIST: + try: + creator_info_parsed = parse_creator_info_from_url(creator_url) + user_id = creator_info_parsed.sec_user_id + utils.logger.info(f"[DouYinCrawler.get_creators_and_videos] Parsed sec_user_id: {user_id} from {creator_url}") + except ValueError as e: + utils.logger.error(f"[DouYinCrawler.get_creators_and_videos] Failed to parse creator URL: {e}") + continue + creator_info: Dict = await self.dy_client.get_user_info(user_id) if creator_info: await douyin_store.save_creator(user_id, creator=creator_info) diff --git a/media_platform/douyin/help.py b/media_platform/douyin/help.py index 1ed3111..d4e245d 100644 --- a/media_platform/douyin/help.py +++ b/media_platform/douyin/help.py @@ -16,10 +16,15 @@ # @Desc : 获取 a_bogus 参数, 学习交流使用，请勿用作商业用途，侵权联系作者删除 import random +import re +from typing import Optional import execjs from playwright.async_api import Page +from model.m_douyin import VideoUrlInfo, CreatorUrlInfo +from tools.crawler_util import extract_url_params_to_dict + douyin_sign_obj = execjs.compile(open('libs/douyin.js', encoding='utf-8-sig').read()) def get_web_id(): @@ -83,3 +88,103 @@ async def get_a_bogus_from_playright(params: str, post_data: dict, user_agent: s return a_bogus + +def parse_video_info_from_url(url: str) -> VideoUrlInfo: + """ + 从抖音视频URL中解析出视频ID + 支持以下格式: + 1. 普通视频链接: https://www.douyin.com/video/7525082444551310602 + 2. 带modal_id参数的链接: + - https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?modal_id=7525082444551310602 + - https://www.douyin.com/root/search/python?modal_id=7471165520058862848 + 3. 短链接: https://v.douyin.com/iF12345ABC/ (需要client解析) + 4. 纯ID: 7525082444551310602 + + Args: + url: 抖音视频链接或ID + Returns: + VideoUrlInfo: 包含视频ID的对象 + """ + # 如果是纯数字ID,直接返回 + if url.isdigit(): + return VideoUrlInfo(aweme_id=url, url_type="normal") + + # 检查是否是短链接 (v.douyin.com) + if "v.douyin.com" in url or url.startswith("http") and len(url) < 50 and "video" not in url: + return VideoUrlInfo(aweme_id="", url_type="short") # 需要通过client解析 + + # 尝试从URL参数中提取modal_id + params = extract_url_params_to_dict(url) + modal_id = params.get("modal_id") + if modal_id: + return VideoUrlInfo(aweme_id=modal_id, url_type="modal") + + # 从标准视频URL中提取ID: /video/数字 + video_pattern = r'/video/(\d+)' + match = re.search(video_pattern, url) + if match: + aweme_id = match.group(1) + return VideoUrlInfo(aweme_id=aweme_id, url_type="normal") + + raise ValueError(f"无法从URL中解析出视频ID: {url}") + + +def parse_creator_info_from_url(url: str) -> CreatorUrlInfo: + """ + 从抖音创作者主页URL中解析出创作者ID (sec_user_id) + 支持以下格式: + 1. 创作者主页: https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main + 2. 纯ID: MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE + + Args: + url: 抖音创作者主页链接或sec_user_id + Returns: + CreatorUrlInfo: 包含创作者ID的对象 + """ + # 如果是纯ID格式(通常以MS4wLjABAAAA开头),直接返回 + if url.startswith("MS4wLjABAAAA") or (not url.startswith("http") and "douyin.com" not in url): + return CreatorUrlInfo(sec_user_id=url) + + # 从创作者主页URL中提取sec_user_id: /user/xxx + user_pattern = r'/user/([^/?]+)' + match = re.search(user_pattern, url) + if match: + sec_user_id = match.group(1) + return CreatorUrlInfo(sec_user_id=sec_user_id) + + raise ValueError(f"无法从URL中解析出创作者ID: {url}") + + +if __name__ == '__main__': + # 测试视频URL解析 + print("=== 视频URL解析测试 ===") + test_urls = [ + "https://www.douyin.com/video/7525082444551310602", + "https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main&modal_id=7525082444551310602", + "https://www.douyin.com/root/search/python?aid=b733a3b0-4662-4639-9a72-c2318fba9f3f&modal_id=7471165520058862848&type=general", + "7525082444551310602", + ] + for url in test_urls: + try: + result = parse_video_info_from_url(url) + print(f"✓ URL: {url[:80]}...") + print(f" 结果: {result}\n") + except Exception as e: + print(f"✗ URL: {url}") + print(f" 错误: {e}\n") + + # 测试创作者URL解析 + print("=== 创作者URL解析测试 ===") + test_creator_urls = [ + "https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main", + "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE", + ] + for url in test_creator_urls: + try: + result = parse_creator_info_from_url(url) + print(f"✓ URL: {url[:80]}...") + print(f" 结果: {result}\n") + except Exception as e: + print(f"✗ URL: {url}") + print(f" 错误: {e}\n") + diff --git a/model/m_douyin.py b/model/m_douyin.py index e907b1d..523399b 100644 --- a/model/m_douyin.py +++ b/model/m_douyin.py @@ -1,12 +1,25 @@ -# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： -# 1. 不得用于任何商业用途。 -# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 -# 3. 不得进行大规模爬取或对平台造成运营干扰。 -# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 # 5. 不得用于任何非法或不当的用途。 -# -# 详细许可条款请参阅项目根目录下的LICENSE文件。 -# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 # -*- coding: utf-8 -*- + +from pydantic import BaseModel, Field + + +class VideoUrlInfo(BaseModel): + """抖音视频URL信息""" + aweme_id: str = Field(title="aweme id (video id)") + url_type: str = Field(default="normal", title="url type: normal, short, modal") + + +class CreatorUrlInfo(BaseModel): + """抖音创作者URL信息""" + sec_user_id: str = Field(title="sec_user_id (creator id)")