feat: 微博爬虫帖子搜索完成

This commit is contained in:
Relakkes
2023-12-24 17:57:48 +08:00
parent 9785abba5d
commit c5b64fdbf5
14 changed files with 671 additions and 19 deletions

View File

@@ -9,7 +9,7 @@ from media_platform.bilibili import BilibiliCrawler
from media_platform.douyin import DouYinCrawler
from media_platform.kuaishou import KuaishouCrawler
from media_platform.xhs import XiaoHongShuCrawler
from proxy import proxy_account_pool
from media_platform.weibo import WeiboCrawler
class CrawlerFactory:
@@ -17,7 +17,8 @@ class CrawlerFactory:
"xhs": XiaoHongShuCrawler,
"dy": DouYinCrawler,
"ks": KuaishouCrawler,
"bili": BilibiliCrawler
"bili": BilibiliCrawler,
"wb": WeiboCrawler
}
@staticmethod
@@ -31,8 +32,8 @@ class CrawlerFactory:
async def main():
# define command line params ...
parser = argparse.ArgumentParser(description='Media crawler program.')
parser.add_argument('--platform', type=str, help='Media platform select (xhs | dy | ks | bili)',
choices=["xhs", "dy", "ks", "bili"], default=config.PLATFORM)
parser.add_argument('--platform', type=str, help='Media platform select (xhs | dy | ks | bili | wb)',
choices=["xhs", "dy", "ks", "bili", "wb"], default=config.PLATFORM)
parser.add_argument('--lt', type=str, help='Login type (qrcode | phone | cookie)',
choices=["qrcode", "phone", "cookie"], default=config.LOGIN_TYPE)
parser.add_argument('--type', type=str, help='crawler type (search | detail)',