290 lines
11 KiB
Python
290 lines
11 KiB
Python
import core.proxy_service as proxy_service_module
|
|
from core.proxy_service import ProxyService
|
|
|
|
|
|
def test_webshare_requests_config_with_country():
|
|
service = ProxyService()
|
|
config = service._webshare_requests(country='gb')
|
|
assert '-gb-rotate' in config['http']
|
|
assert config['http'] == config['https']
|
|
|
|
|
|
def test_webshare_requests_config_lowercases_uppercase_country():
|
|
# competitors.primary_market is stored as an uppercase ISO code
|
|
# (CLAUDE.md's schema — "GB", "US", "AU"), but Webshare's rotating
|
|
# endpoint only recognises lowercase country codes in the username
|
|
# (help.webshare.io/en/articles/8596715 — examples are `-us`, `-gb`,
|
|
# `-ca`, never uppercase). An uppercase suffix here previously caused
|
|
# every proxied scrape to fail Webshare auth silently.
|
|
service = ProxyService()
|
|
config = service._webshare_requests(country='GB')
|
|
assert '-gb-rotate' in config['http']
|
|
assert '-GB-rotate' not in config['http']
|
|
|
|
|
|
def test_webshare_requests_config_without_country():
|
|
service = ProxyService()
|
|
config = service._webshare_requests()
|
|
assert '-rotate' in config['http']
|
|
assert '-gb-' not in config['http']
|
|
|
|
|
|
def test_webshare_playwright_config():
|
|
service = ProxyService()
|
|
config = service._webshare_playwright(country='us')
|
|
assert config['server'] == 'http://p.webshare.io:80'
|
|
assert config['username'].endswith('-us-rotate')
|
|
|
|
|
|
def test_webshare_playwright_config_lowercases_uppercase_country():
|
|
service = ProxyService()
|
|
config = service._webshare_playwright(country='US')
|
|
assert config['username'].endswith('-us-rotate')
|
|
|
|
|
|
def test_brightdata_requests_and_playwright_configs():
|
|
# Port 33335, not the commonly-documented 22225 — Bright Data's currently
|
|
# valid SSL cert requires 33335; 22225 is the deprecated cert's port,
|
|
# which produced a real net::ERR_CERT_AUTHORITY_INVALID on a live scrape.
|
|
service = ProxyService()
|
|
assert service._brightdata_requests()['http'].startswith('http://')
|
|
assert service._brightdata_playwright()['server'] == 'http://brd.superproxy.io:33335'
|
|
|
|
|
|
def test_get_proxy_for_domain_uses_webshare_when_no_override(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None)
|
|
config = service.get_proxy_for_domain('intrepidtravel.com', country='GB')
|
|
assert 'p.webshare.io' in config['http']
|
|
|
|
|
|
def test_get_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr(
|
|
'core.proxy_service.proxy_repository.get_by_domain',
|
|
lambda domain: {'id': 'o1', 'provider': 'brightdata'}
|
|
)
|
|
config = service.get_proxy_for_domain('flashpack.com')
|
|
assert 'brd.superproxy.io' in config['http']
|
|
|
|
|
|
def test_get_playwright_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr(
|
|
'core.proxy_service.proxy_repository.get_by_domain',
|
|
lambda domain: {'id': 'o1', 'provider': 'brightdata'}
|
|
)
|
|
config = service.get_playwright_proxy_for_domain('flashpack.com')
|
|
assert config['server'] == 'http://brd.superproxy.io:33335'
|
|
|
|
|
|
def test_escalate_to_brightdata_creates_new_override(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None)
|
|
created = {}
|
|
monkeypatch.setattr('core.proxy_service.proxy_repository.create', lambda data: created.update(data))
|
|
|
|
service.escalate_to_brightdata('newlyblocked.com')
|
|
|
|
assert created['domain'] == 'newlyblocked.com'
|
|
assert created['provider'] == 'brightdata'
|
|
assert created['blocked_count'] == 1
|
|
|
|
|
|
def test_escalate_to_brightdata_increments_existing_override(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr(
|
|
'core.proxy_service.proxy_repository.get_by_domain',
|
|
lambda domain: {'id': 'o1', 'blocked_count': 2}
|
|
)
|
|
updated = {}
|
|
monkeypatch.setattr(
|
|
'core.proxy_service.proxy_repository.update',
|
|
lambda override_id, data: updated.update({'id': override_id, **data})
|
|
)
|
|
|
|
service.escalate_to_brightdata('repeatoffender.com')
|
|
|
|
assert updated['id'] == 'o1'
|
|
assert updated['blocked_count'] == 3
|
|
assert updated['provider'] == 'brightdata'
|
|
|
|
|
|
# ── is_domain_blocked_on_webshare ────────────────────────────────────────
|
|
|
|
def test_is_domain_blocked_on_webshare_true_when_brightdata_override(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr(
|
|
'core.proxy_service.proxy_repository.get_by_domain',
|
|
lambda domain: {'id': 'o1', 'provider': 'brightdata'}
|
|
)
|
|
assert service.is_domain_blocked_on_webshare('flashpack.com') is True
|
|
|
|
|
|
def test_is_domain_blocked_on_webshare_false_when_no_override(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None)
|
|
assert service.is_domain_blocked_on_webshare('intrepidtravel.com') is False
|
|
|
|
|
|
def test_is_domain_blocked_on_webshare_false_when_webshare_override(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr(
|
|
'core.proxy_service.proxy_repository.get_by_domain',
|
|
lambda domain: {'id': 'o1', 'provider': 'webshare'}
|
|
)
|
|
assert service.is_domain_blocked_on_webshare('intrepidtravel.com') is False
|
|
|
|
|
|
# ── get_webshare_ip_candidates / _fetch_webshare_ip_list ─────────────────
|
|
|
|
class _FakeResponse:
|
|
def __init__(self, json_data, status_code=200):
|
|
self._json_data = json_data
|
|
self.status_code = status_code
|
|
|
|
def raise_for_status(self):
|
|
if self.status_code >= 400:
|
|
raise proxy_service_module.requests.HTTPError(f'{self.status_code}')
|
|
|
|
def json(self):
|
|
return self._json_data
|
|
|
|
|
|
def _fake_ip(suffix, valid=True, country='GB'):
|
|
return {
|
|
'id': f'd-{suffix}', 'username': f'user{suffix}', 'password': f'pass{suffix}',
|
|
'proxy_address': f'1.2.3.{suffix}', 'port': 8000 + suffix,
|
|
'valid': valid, 'country_code': country,
|
|
}
|
|
|
|
|
|
def test_get_webshare_ip_candidates_returns_empty_without_country():
|
|
service = ProxyService()
|
|
assert service.get_webshare_ip_candidates(None) == []
|
|
|
|
|
|
def test_fetch_webshare_ip_list_sends_uppercase_country_code(monkeypatch):
|
|
# Webshare's proxy-list *management* API wants uppercase country codes
|
|
# (their own docs example: "FR,US") — the opposite convention from the
|
|
# rotating-gateway username suffix, which wants lowercase. These are two
|
|
# separate Webshare systems; do not "fix" them to match each other.
|
|
service = ProxyService()
|
|
captured = {}
|
|
|
|
def fake_get(url, headers=None, params=None, timeout=None):
|
|
captured['params'] = params
|
|
return _FakeResponse({'results': [_fake_ip(1)], 'next': None})
|
|
|
|
monkeypatch.setattr(proxy_service_module.requests, 'get', fake_get)
|
|
service.get_webshare_ip_candidates('gb')
|
|
|
|
assert captured['params']['country_code__in'] == 'GB'
|
|
|
|
|
|
def test_get_webshare_ip_candidates_filters_valid_and_respects_limit(monkeypatch):
|
|
service = ProxyService()
|
|
ips = [_fake_ip(1), _fake_ip(2, valid=False), _fake_ip(3), _fake_ip(4), _fake_ip(5)]
|
|
|
|
monkeypatch.setattr(
|
|
proxy_service_module.requests, 'get',
|
|
lambda *a, **k: _FakeResponse({'results': ips, 'next': None})
|
|
)
|
|
|
|
candidates = service.get_webshare_ip_candidates('GB', limit=2)
|
|
|
|
assert len(candidates) == 2
|
|
# The invalid one (suffix 2) must never appear, since it's filtered
|
|
# before the random.sample() call — every candidate has real fields.
|
|
for c in candidates:
|
|
assert 'password' in c and c['password'] != 'pass2'
|
|
|
|
|
|
def test_get_webshare_ip_candidates_dict_shape_uses_per_ip_credentials(monkeypatch):
|
|
# Each Webshare list entry carries its OWN username/password/port —
|
|
# genuinely different from _webshare_playwright()'s single account-level
|
|
# rotating-gateway credentials at p.webshare.io:80.
|
|
service = ProxyService()
|
|
monkeypatch.setattr(
|
|
proxy_service_module.requests, 'get',
|
|
lambda *a, **k: _FakeResponse({'results': [_fake_ip(7)], 'next': None})
|
|
)
|
|
|
|
candidates = service.get_webshare_ip_candidates('GB', limit=1)
|
|
|
|
assert candidates == [{'server': 'http://1.2.3.7:8007', 'username': 'user7', 'password': 'pass7'}]
|
|
|
|
|
|
def test_get_webshare_ip_candidates_returns_empty_on_api_failure(monkeypatch):
|
|
service = ProxyService()
|
|
|
|
def raising_get(*a, **k):
|
|
raise proxy_service_module.requests.RequestException('connection refused')
|
|
|
|
monkeypatch.setattr(proxy_service_module.requests, 'get', raising_get)
|
|
assert service.get_webshare_ip_candidates('GB') == []
|
|
|
|
|
|
def test_get_webshare_ip_candidates_returns_empty_when_no_results_for_country(monkeypatch):
|
|
service = ProxyService()
|
|
monkeypatch.setattr(
|
|
proxy_service_module.requests, 'get',
|
|
lambda *a, **k: _FakeResponse({'results': [], 'next': None})
|
|
)
|
|
assert service.get_webshare_ip_candidates('GB') == []
|
|
|
|
|
|
def test_fetch_webshare_ip_list_caches_within_ttl(monkeypatch):
|
|
service = ProxyService()
|
|
call_count = {'n': 0}
|
|
|
|
def fake_get(*a, **k):
|
|
call_count['n'] += 1
|
|
return _FakeResponse({'results': [_fake_ip(1)], 'next': None})
|
|
|
|
monkeypatch.setattr(proxy_service_module.requests, 'get', fake_get)
|
|
monkeypatch.setattr(proxy_service_module.time, 'time', lambda: 1000.0)
|
|
|
|
service.get_webshare_ip_candidates('GB')
|
|
service.get_webshare_ip_candidates('GB')
|
|
|
|
assert call_count['n'] == 1 # second call served from cache
|
|
|
|
|
|
def test_fetch_webshare_ip_list_refetches_after_ttl_expires(monkeypatch):
|
|
service = ProxyService()
|
|
call_count = {'n': 0}
|
|
current_time = {'t': 1000.0}
|
|
|
|
def fake_get(*a, **k):
|
|
call_count['n'] += 1
|
|
return _FakeResponse({'results': [_fake_ip(1)], 'next': None})
|
|
|
|
monkeypatch.setattr(proxy_service_module.requests, 'get', fake_get)
|
|
monkeypatch.setattr(proxy_service_module.time, 'time', lambda: current_time['t'])
|
|
|
|
service.get_webshare_ip_candidates('GB')
|
|
current_time['t'] += proxy_service_module.WEBSHARE_IP_CACHE_TTL_SECONDS + 1
|
|
service.get_webshare_ip_candidates('GB')
|
|
|
|
assert call_count['n'] == 2
|
|
|
|
|
|
def test_fetch_webshare_ip_list_negative_cache_uses_shorter_ttl(monkeypatch):
|
|
service = ProxyService()
|
|
call_count = {'n': 0}
|
|
current_time = {'t': 1000.0}
|
|
|
|
def raising_get(*a, **k):
|
|
call_count['n'] += 1
|
|
raise proxy_service_module.requests.RequestException('down')
|
|
|
|
monkeypatch.setattr(proxy_service_module.requests, 'get', raising_get)
|
|
monkeypatch.setattr(proxy_service_module.time, 'time', lambda: current_time['t'])
|
|
|
|
service.get_webshare_ip_candidates('GB')
|
|
current_time['t'] += proxy_service_module.WEBSHARE_IP_CACHE_NEGATIVE_TTL_SECONDS + 1
|
|
service.get_webshare_ip_candidates('GB')
|
|
|
|
assert call_count['n'] == 2 # negative cache expired, retried
|