104 lines
4.2 KiB
Python
104 lines
4.2 KiB
Python
import os
|
|
from datetime import datetime, timezone
|
|
from repositories.proxy_repository import proxy_repository
|
|
|
|
WEBSHARE_USER = os.getenv('WEBSHARE_USER')
|
|
WEBSHARE_PASS = os.getenv('WEBSHARE_PASS')
|
|
BRIGHTDATA_USER = os.getenv('BRIGHTDATA_USER')
|
|
BRIGHTDATA_PASS = os.getenv('BRIGHTDATA_PASS')
|
|
|
|
|
|
class ProxyService:
|
|
"""
|
|
Smart proxy provider — uses Webshare by default with country
|
|
targeting, auto-escalates to Bright Data for domains that block
|
|
Webshare. This is the ONLY place in the codebase that knows about
|
|
any proxy provider — swapping providers or switching logic means
|
|
changing only this file (CLAUDE.md §10).
|
|
|
|
Country targeting uses competitors.primary_market (ISO code e.g.
|
|
GB, US, AU) passed through from the competitor record — no
|
|
hardcoded country map in code.
|
|
|
|
Data flow (first request to a domain):
|
|
domain + country → proxy_overrides lookup → no override found →
|
|
Webshare config with country targeting returned →
|
|
scraper attempts request → if blocked →
|
|
escalate_to_brightdata(domain) called →
|
|
Bright Data config returned → scraper retries →
|
|
override stored in PocketBase for this domain
|
|
|
|
Data flow (subsequent requests to a known blocked domain):
|
|
domain + country → proxy_overrides lookup → brightdata override
|
|
found → Bright Data config returned immediately, no Webshare attempt
|
|
"""
|
|
|
|
def get_proxy_for_domain(self, domain, country=None):
|
|
"""Returns a `requests`-style proxy config, routed to Bright Data if previously blocked."""
|
|
override = proxy_repository.get_by_domain(domain)
|
|
if override and override.get('provider') == 'brightdata':
|
|
return self._brightdata_requests()
|
|
return self._webshare_requests(country=country)
|
|
|
|
def get_playwright_proxy_for_domain(self, domain, country=None):
|
|
"""Returns a Playwright proxy config, routed to Bright Data if previously blocked."""
|
|
override = proxy_repository.get_by_domain(domain)
|
|
if override and override.get('provider') == 'brightdata':
|
|
return self._brightdata_playwright()
|
|
return self._webshare_playwright(country=country)
|
|
|
|
def escalate_to_brightdata(self, domain):
|
|
"""
|
|
Records that Webshare was blocked on this domain. Upserts a
|
|
proxy_override so future requests skip Webshare entirely for
|
|
this domain.
|
|
"""
|
|
existing = proxy_repository.get_by_domain(domain)
|
|
if existing:
|
|
proxy_repository.update(existing['id'], {
|
|
'provider': 'brightdata',
|
|
'blocked_count': existing.get('blocked_count', 0) + 1,
|
|
'last_blocked': datetime.now(timezone.utc).isoformat(),
|
|
})
|
|
else:
|
|
proxy_repository.create({
|
|
'domain': domain,
|
|
'provider': 'brightdata',
|
|
'blocked_count': 1,
|
|
'last_blocked': datetime.now(timezone.utc).isoformat(),
|
|
})
|
|
|
|
def _webshare_requests(self, country=None):
|
|
"""Webshare `requests` proxy config. Format: username-{COUNTRY}-rotate. No country = full pool rotation."""
|
|
suffix = f'-{country.upper()}-rotate' if country else '-rotate'
|
|
username = f'{WEBSHARE_USER}{suffix}'
|
|
return {
|
|
'http': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80',
|
|
'https': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80',
|
|
}
|
|
|
|
def _webshare_playwright(self, country=None):
|
|
"""Webshare Playwright proxy config with optional country targeting."""
|
|
suffix = f'-{country.upper()}-rotate' if country else '-rotate'
|
|
return {
|
|
'server': 'http://p.webshare.io:80',
|
|
'username': f'{WEBSHARE_USER}{suffix}',
|
|
'password': WEBSHARE_PASS,
|
|
}
|
|
|
|
def _brightdata_requests(self):
|
|
return {
|
|
'http': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:22225',
|
|
'https': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:22225',
|
|
}
|
|
|
|
def _brightdata_playwright(self):
|
|
return {
|
|
'server': 'http://brd.superproxy.io:22225',
|
|
'username': BRIGHTDATA_USER,
|
|
'password': BRIGHTDATA_PASS,
|
|
}
|
|
|
|
|
|
proxy_service = ProxyService()
|