diff --git a/config/base_config.py b/config/base_config.py index 49c35ce..b2f084d 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -34,10 +34,14 @@ CRAWLER_TYPE = ( ENABLE_IP_PROXY = False # Number of proxy IP pools -IP_PROXY_POOL_COUNT = 2 +IP_PROXY_POOL_COUNT = 1 # Proxy IP provider name -IP_PROXY_PROVIDER_NAME = "kuaidaili" # kuaidaili | wandouhttp +IP_PROXY_PROVIDER_NAME = "static" # kuaidaili | wandouhttp | static + +# Static proxy configuration (used when IP_PROXY_PROVIDER_NAME is set to "static") +# Format: "http://your_home_domain:port" or "http://user:password@your_home_domain:port" +STATIC_PROXY_URL = "http://your_home_domain:port" # Setting to True will not open the browser (headless browser) # Setting False will open a browser diff --git a/proxy/proxy_ip_pool.py b/proxy/proxy_ip_pool.py index 8f4a129..4970b28 100644 --- a/proxy/proxy_ip_pool.py +++ b/proxy/proxy_ip_pool.py @@ -150,9 +150,46 @@ class ProxyIpPool: await self.load_proxies() +class StaticProxyProvider(ProxyProvider): + async def get_proxy(self, num: int) -> List[IpInfoModel]: + from urllib.parse import urlparse + import time + + proxy_url = getattr(config, "STATIC_PROXY_URL", "") + if not proxy_url: + utils.logger.warning("[StaticProxyProvider] STATIC_PROXY_URL is not configured!") + return [] + + try: + parsed = urlparse(proxy_url) + ip = parsed.hostname or "" + port = parsed.port or 80 + user = parsed.username or "" + password = parsed.password or "" + protocol = parsed.scheme + "://" if parsed.scheme else "http://" + + # Static proxy doesn't expire + expired_time_ts = int(time.time()) + 99999999 + + return [ + IpInfoModel( + ip=ip, + port=port, + user=user, + password=password, + protocol=protocol, + expired_time_ts=expired_time_ts + ) + ] + except Exception as e: + utils.logger.error(f"[StaticProxyProvider] Parse static proxy url error: {e}") + return [] + + IpProxyProvider: Dict[str, ProxyProvider] = { ProviderNameEnum.KUAI_DAILI_PROVIDER.value: new_kuai_daili_proxy(), ProviderNameEnum.WANDOU_HTTP_PROVIDER.value: new_wandou_http_proxy(), + "static": StaticProxyProvider(), }