jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-02-06 15:11:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 优化代码

Browse Source
This commit is contained in:
Relakkes
2023-07-29 15:35:40 +08:00
parent febbb133d7
commit 4ff2cf8661
17 changed files with 133 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
main.py
View File

@@ -1,9 +1,9 @@
import sys
import asyncio
import argparse
import asyncio
import sys
import db
import config
import db
from base import proxy_account_pool
from media_platform.douyin import DouYinCrawler
from media_platform.xhs import XiaoHongShuCrawler
@@ -17,14 +17,16 @@ class CrawlerFactory:
elif platform == "dy":
return DouYinCrawler()
else:
raise ValueError("Invalid Media Platform Currently only supported xhs or douyin ...")
raise ValueError("Invalid Media Platform Currently only supported xhs or dy ...")
async def main():
# define command line params ...
parser = argparse.ArgumentParser(description='Media crawler program.')
parser.add_argument('--platform', type=str, help='Media platform select (xhs|dy)...', default=config.PLATFORM)
parser.add_argument('--lt', type=str, help='Login type (qrcode | phone | cookie)', default=config.LOGIN_TYPE)
parser.add_argument('--platform', type=str, help='Media platform select (xhs|dy)', choices=["xhs", "dy"],
default=config.PLATFORM)
parser.add_argument('--lt', type=str, help='Login type (qrcode | phone | cookie)',
choices=["qrcode", "phone", "cookie"], default=config.LOGIN_TYPE)
# init account pool
account_pool = proxy_account_pool.create_account_pool()
@@ -34,9 +36,10 @@ async def main():
await db.init_db()
args = parser.parse_args()
crawler = CrawlerFactory().create_crawler(platform=args.platform)
crawler = CrawlerFactory.create_crawler(platform=args.platform)
crawler.init_config(
command_args=args,
platform=args.platform,
login_type=args.lt,
account_pool=account_pool
)
await crawler.start()
@@ -44,6 +47,7 @@ async def main():
if __name__ == '__main__':
try:
asyncio.run(main())
# asyncio.run(main())
asyncio.get_event_loop().run_until_complete(main())
except KeyboardInterrupt:
sys.exit()