259 lines
12 KiB
Python
259 lines
12 KiB
Python
import os
|
|
import time
|
|
import random
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
import requests
|
|
from repositories.proxy_repository import proxy_repository
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
WEBSHARE_USER = os.getenv('WEBSHARE_USER')
|
|
WEBSHARE_PASS = os.getenv('WEBSHARE_PASS')
|
|
# Separate from WEBSHARE_USER/WEBSHARE_PASS (proxy-gateway auth) — this is a
|
|
# Webshare *dashboard API token* (Authorization: Token <key>) for their
|
|
# account-management REST API, used only by _fetch_webshare_ip_list() to
|
|
# list the account's own dedicated IPs. Generated from the Webshare
|
|
# dashboard's API/account section, not the proxy connection settings page.
|
|
WEBSHARE_API_KEY = os.getenv('WEBSHARE_API_KEY')
|
|
BRIGHTDATA_USER = os.getenv('BRIGHTDATA_USER')
|
|
BRIGHTDATA_PASS = os.getenv('BRIGHTDATA_PASS')
|
|
|
|
WEBSHARE_PROXY_LIST_URL = 'https://proxy.webshare.io/api/v2/proxy/list/'
|
|
# Bounds latency (each blocked attempt costs a full Playwright render cycle,
|
|
# up to _render_with_proxy()'s own 30-60s timeouts) while still being enough
|
|
# to distinguish "one flagged IP" (fixed by 2-3 alternates from the same
|
|
# account) from "this site blocks the whole IP range" — no number of
|
|
# same-account IPs fixes that; only Bright Data's different network does.
|
|
WEBSHARE_IP_CANDIDATE_LIMIT = 3
|
|
WEBSHARE_IP_CACHE_TTL_SECONDS = 1800 # 30 min — the account's dedicated IPs rarely change
|
|
# Short on purpose: bounds a retry storm within one batch run (many domains
|
|
# targeting the same country back-to-back) without permanently caching what
|
|
# might be a transient failure (bad token, momentary API outage).
|
|
WEBSHARE_IP_CACHE_NEGATIVE_TTL_SECONDS = 300 # 5 min
|
|
|
|
|
|
class ProxyService:
|
|
"""
|
|
Smart proxy provider — uses Webshare by default with country
|
|
targeting, auto-escalates to Bright Data for domains that block
|
|
Webshare. This is the ONLY place in the codebase that knows about
|
|
any proxy provider — swapping providers or switching logic means
|
|
changing only this file (CLAUDE.md §10).
|
|
|
|
Country targeting uses competitors.primary_market (ISO code e.g.
|
|
GB, US, AU) passed through from the competitor record — no
|
|
hardcoded country map in code.
|
|
|
|
Data flow (first request to a domain):
|
|
domain + country → proxy_overrides lookup → no override found →
|
|
Webshare config with country targeting returned →
|
|
scraper attempts request → if blocked →
|
|
escalate_to_brightdata(domain) called →
|
|
Bright Data config returned → scraper retries →
|
|
override stored in PocketBase for this domain
|
|
|
|
Data flow (subsequent requests to a known blocked domain):
|
|
domain + country → proxy_overrides lookup → brightdata override
|
|
found → Bright Data config returned immediately, no Webshare attempt
|
|
|
|
Data flow (specific-IP candidate rotation — before ever escalating to
|
|
Bright Data): domain not already known-blocked → the account's own
|
|
dedicated Webshare IPs for this country fetched (cached) via
|
|
get_webshare_ip_candidates() → core.scraper.render_page() tries a
|
|
handful of those specific IPs in turn → only once all of them are
|
|
exhausted (or none were available) does it fall through to the single
|
|
`-rotate` gateway attempt and then Bright Data, as before.
|
|
"""
|
|
|
|
def __init__(self):
|
|
# In-memory cache of the account's dedicated Webshare IPs, keyed by
|
|
# uppercased country code — {'expires_at': float, 'results': [...]}.
|
|
# Same time.time()-based pattern as repositories/pocketbase_client.py's
|
|
# token cache. Not persisted anywhere — a fresh process just refetches.
|
|
self._ip_cache = {}
|
|
|
|
def get_proxy_for_domain(self, domain, country=None):
|
|
"""Returns a `requests`-style proxy config, routed to Bright Data if previously blocked."""
|
|
override = proxy_repository.get_by_domain(domain)
|
|
if override and override.get('provider') == 'brightdata':
|
|
return self._brightdata_requests()
|
|
return self._webshare_requests(country=country)
|
|
|
|
def get_playwright_proxy_for_domain(self, domain, country=None):
|
|
"""Returns a Playwright proxy config, routed to Bright Data if previously blocked."""
|
|
override = proxy_repository.get_by_domain(domain)
|
|
if override and override.get('provider') == 'brightdata':
|
|
return self._brightdata_playwright()
|
|
return self._webshare_playwright(country=country)
|
|
|
|
def is_domain_blocked_on_webshare(self, domain):
|
|
"""
|
|
Returns True if this domain already has a `brightdata` override —
|
|
i.e. Webshare (gateway or specific IPs) is already known not to
|
|
work here. Lets core.scraper.render_page() skip the specific-IP
|
|
candidate loop entirely for a domain that's already established as
|
|
Bright-Data-only, rather than re-running a 3-IP check every single
|
|
scrape once that's settled. Thin wrapper so scraper.py never reaches
|
|
into proxy_repository directly — this class stays the only place
|
|
that knows about proxy providers.
|
|
"""
|
|
override = proxy_repository.get_by_domain(domain)
|
|
return bool(override and override.get('provider') == 'brightdata')
|
|
|
|
def get_webshare_ip_candidates(self, country, limit=WEBSHARE_IP_CANDIDATE_LIMIT):
|
|
"""
|
|
Returns up to `limit` Playwright proxy configs for specific
|
|
dedicated Webshare IPs assigned to this account, filtered to the
|
|
target country. Returns [] if none are available (no country given,
|
|
API failure, or no results) — core.scraper.render_page() treats an
|
|
empty list as "skip straight to the existing single-gateway attempt",
|
|
so this always fails gracefully rather than raising.
|
|
|
|
Each candidate carries its OWN username/password/port from
|
|
Webshare's proxy-list API response — genuinely different from
|
|
_webshare_playwright()'s single account-level rotating-gateway
|
|
credentials, not a variant of it.
|
|
"""
|
|
if not country:
|
|
return []
|
|
results = self._fetch_webshare_ip_list(country)
|
|
if not results:
|
|
return []
|
|
chosen = random.sample(results, min(limit, len(results)))
|
|
return [
|
|
{
|
|
'server': f'http://{ip["proxy_address"]}:{ip["port"]}',
|
|
'username': ip['username'],
|
|
'password': ip['password'],
|
|
}
|
|
for ip in chosen
|
|
]
|
|
|
|
def _fetch_webshare_ip_list(self, country):
|
|
"""
|
|
Fetches (and caches) the account's own dedicated Webshare IPs for a
|
|
country via Webshare's proxy-list *management* API — a different
|
|
host/credential entirely from the proxy gateway (p.webshare.io +
|
|
WEBSHARE_USER/PASS). This one is proxy.webshare.io, authenticated
|
|
with a dashboard API token (WEBSHARE_API_KEY), and it wants the
|
|
country code UPPERCASE (Webshare's own docs example: "FR,US") — the
|
|
opposite case convention from _webshare_playwright()'s gateway
|
|
username suffix, which wants lowercase. These are two unrelated
|
|
Webshare systems with inconsistent casing on their end, not a
|
|
mistake here — do not "fix" them into matching each other.
|
|
|
|
Data flow:
|
|
country → cache hit (within TTL, success or negative) → cached
|
|
list returned →
|
|
cache miss → GET proxy.webshare.io/api/v2/proxy/list/ (mode=direct,
|
|
country_code__in=<COUNTRY>, page_size=100), paginated via `next` →
|
|
filter to valid=true → cache with the long TTL → list returned →
|
|
on any request/HTTP/JSON failure → logged, cached as [] with the
|
|
short negative TTL → [] returned
|
|
"""
|
|
cache_key = country.upper()
|
|
cached = self._ip_cache.get(cache_key)
|
|
if cached and time.time() < cached['expires_at']:
|
|
return cached['results']
|
|
|
|
results = []
|
|
try:
|
|
url = WEBSHARE_PROXY_LIST_URL
|
|
params = {'mode': 'direct', 'country_code__in': cache_key, 'page_size': 100}
|
|
for _ in range(3): # defensive pagination cap — a ~100-IP account should fit in one page
|
|
response = requests.get(
|
|
url, headers={'Authorization': f'Token {WEBSHARE_API_KEY}'},
|
|
params=params, timeout=10
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
results.extend(r for r in data.get('results', []) if r.get('valid'))
|
|
url = data.get('next')
|
|
params = None # `next` already carries its own query string
|
|
if not url:
|
|
break
|
|
self._ip_cache[cache_key] = {
|
|
'expires_at': time.time() + WEBSHARE_IP_CACHE_TTL_SECONDS,
|
|
'results': results,
|
|
}
|
|
except (requests.RequestException, ValueError) as e:
|
|
logger.warning(f'Webshare proxy list fetch failed for {country}: {str(e)[:200]}')
|
|
self._ip_cache[cache_key] = {
|
|
'expires_at': time.time() + WEBSHARE_IP_CACHE_NEGATIVE_TTL_SECONDS,
|
|
'results': [],
|
|
}
|
|
return []
|
|
|
|
return results
|
|
|
|
def escalate_to_brightdata(self, domain):
|
|
"""
|
|
Records that Webshare was blocked on this domain. Upserts a
|
|
proxy_override so future requests skip Webshare entirely for
|
|
this domain.
|
|
"""
|
|
existing = proxy_repository.get_by_domain(domain)
|
|
if existing:
|
|
proxy_repository.update(existing['id'], {
|
|
'provider': 'brightdata',
|
|
'blocked_count': existing.get('blocked_count', 0) + 1,
|
|
'last_blocked': datetime.now(timezone.utc).isoformat(),
|
|
})
|
|
else:
|
|
proxy_repository.create({
|
|
'domain': domain,
|
|
'provider': 'brightdata',
|
|
'blocked_count': 1,
|
|
'last_blocked': datetime.now(timezone.utc).isoformat(),
|
|
})
|
|
|
|
def _webshare_requests(self, country=None):
|
|
"""
|
|
Webshare `requests` proxy config. Format: username-{country}-rotate.
|
|
Country code must be lowercase — Webshare's own rotating-endpoint
|
|
docs (help.webshare.io/en/articles/8596715) show lowercase examples
|
|
only (`-us`, `-gb`, `-ca`); an uppercase code is not a recognised
|
|
country token and fails proxy auth. `country` arrives from
|
|
competitors.primary_market, which is stored uppercase (CLAUDE.md's
|
|
schema — "GB", "US", "AU"), so this always lowercases regardless of
|
|
the input's case. No country = full pool rotation.
|
|
"""
|
|
suffix = f'-{country.lower()}-rotate' if country else '-rotate'
|
|
username = f'{WEBSHARE_USER}{suffix}'
|
|
return {
|
|
'http': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80',
|
|
'https': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80',
|
|
}
|
|
|
|
def _webshare_playwright(self, country=None):
|
|
"""Webshare Playwright proxy config with optional country targeting — see _webshare_requests() for why the country code is lowercased."""
|
|
suffix = f'-{country.lower()}-rotate' if country else '-rotate'
|
|
return {
|
|
'server': 'http://p.webshare.io:80',
|
|
'username': f'{WEBSHARE_USER}{suffix}',
|
|
'password': WEBSHARE_PASS,
|
|
}
|
|
|
|
def _brightdata_requests(self):
|
|
# Port 33335, not the commonly-documented 22225 — Bright Data's
|
|
# currently valid SSL certificate requires 33335; 22225 is tied to
|
|
# their deprecated cert (phased out ahead of its 2026 expiry), which
|
|
# is why a live scrape hit net::ERR_CERT_AUTHORITY_INVALID on this
|
|
# host before this fix.
|
|
return {
|
|
'http': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:33335',
|
|
'https': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:33335',
|
|
}
|
|
|
|
def _brightdata_playwright(self):
|
|
# See _brightdata_requests() — port 33335 required for the current cert.
|
|
return {
|
|
'server': 'http://brd.superproxy.io:33335',
|
|
'username': BRIGHTDATA_USER,
|
|
'password': BRIGHTDATA_PASS,
|
|
}
|
|
|
|
|
|
proxy_service = ProxyService()
|