import core.proxy_service as proxy_service_module from core.proxy_service import ProxyService def test_webshare_requests_config_with_country(): service = ProxyService() config = service._webshare_requests(country='gb') assert '-gb-rotate' in config['http'] assert config['http'] == config['https'] def test_webshare_requests_config_lowercases_uppercase_country(): # competitors.primary_market is stored as an uppercase ISO code # (CLAUDE.md's schema — "GB", "US", "AU"), but Webshare's rotating # endpoint only recognises lowercase country codes in the username # (help.webshare.io/en/articles/8596715 — examples are `-us`, `-gb`, # `-ca`, never uppercase). An uppercase suffix here previously caused # every proxied scrape to fail Webshare auth silently. service = ProxyService() config = service._webshare_requests(country='GB') assert '-gb-rotate' in config['http'] assert '-GB-rotate' not in config['http'] def test_webshare_requests_config_without_country(): service = ProxyService() config = service._webshare_requests() assert '-rotate' in config['http'] assert '-gb-' not in config['http'] def test_webshare_playwright_config(): service = ProxyService() config = service._webshare_playwright(country='us') assert config['server'] == 'http://p.webshare.io:80' assert config['username'].endswith('-us-rotate') def test_webshare_playwright_config_lowercases_uppercase_country(): service = ProxyService() config = service._webshare_playwright(country='US') assert config['username'].endswith('-us-rotate') def test_brightdata_requests_and_playwright_configs(): # Port 33335, not the commonly-documented 22225 — Bright Data's currently # valid SSL cert requires 33335; 22225 is the deprecated cert's port, # which produced a real net::ERR_CERT_AUTHORITY_INVALID on a live scrape. service = ProxyService() assert service._brightdata_requests()['http'].startswith('http://') assert service._brightdata_playwright()['server'] == 'http://brd.superproxy.io:33335' def test_get_proxy_for_domain_uses_webshare_when_no_override(monkeypatch): service = ProxyService() monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None) config = service.get_proxy_for_domain('intrepidtravel.com', country='GB') assert 'p.webshare.io' in config['http'] def test_get_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch): service = ProxyService() monkeypatch.setattr( 'core.proxy_service.proxy_repository.get_by_domain', lambda domain: {'id': 'o1', 'provider': 'brightdata'} ) config = service.get_proxy_for_domain('flashpack.com') assert 'brd.superproxy.io' in config['http'] def test_get_playwright_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch): service = ProxyService() monkeypatch.setattr( 'core.proxy_service.proxy_repository.get_by_domain', lambda domain: {'id': 'o1', 'provider': 'brightdata'} ) config = service.get_playwright_proxy_for_domain('flashpack.com') assert config['server'] == 'http://brd.superproxy.io:33335' def test_escalate_to_brightdata_creates_new_override(monkeypatch): service = ProxyService() monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None) created = {} monkeypatch.setattr('core.proxy_service.proxy_repository.create', lambda data: created.update(data)) service.escalate_to_brightdata('newlyblocked.com') assert created['domain'] == 'newlyblocked.com' assert created['provider'] == 'brightdata' assert created['blocked_count'] == 1 def test_escalate_to_brightdata_increments_existing_override(monkeypatch): service = ProxyService() monkeypatch.setattr( 'core.proxy_service.proxy_repository.get_by_domain', lambda domain: {'id': 'o1', 'blocked_count': 2} ) updated = {} monkeypatch.setattr( 'core.proxy_service.proxy_repository.update', lambda override_id, data: updated.update({'id': override_id, **data}) ) service.escalate_to_brightdata('repeatoffender.com') assert updated['id'] == 'o1' assert updated['blocked_count'] == 3 assert updated['provider'] == 'brightdata' # ── is_domain_blocked_on_webshare ──────────────────────────────────────── def test_is_domain_blocked_on_webshare_true_when_brightdata_override(monkeypatch): service = ProxyService() monkeypatch.setattr( 'core.proxy_service.proxy_repository.get_by_domain', lambda domain: {'id': 'o1', 'provider': 'brightdata'} ) assert service.is_domain_blocked_on_webshare('flashpack.com') is True def test_is_domain_blocked_on_webshare_false_when_no_override(monkeypatch): service = ProxyService() monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None) assert service.is_domain_blocked_on_webshare('intrepidtravel.com') is False def test_is_domain_blocked_on_webshare_false_when_webshare_override(monkeypatch): service = ProxyService() monkeypatch.setattr( 'core.proxy_service.proxy_repository.get_by_domain', lambda domain: {'id': 'o1', 'provider': 'webshare'} ) assert service.is_domain_blocked_on_webshare('intrepidtravel.com') is False # ── get_webshare_ip_candidates / _fetch_webshare_ip_list ───────────────── class _FakeResponse: def __init__(self, json_data, status_code=200): self._json_data = json_data self.status_code = status_code def raise_for_status(self): if self.status_code >= 400: raise proxy_service_module.requests.HTTPError(f'{self.status_code}') def json(self): return self._json_data def _fake_ip(suffix, valid=True, country='GB'): return { 'id': f'd-{suffix}', 'username': f'user{suffix}', 'password': f'pass{suffix}', 'proxy_address': f'1.2.3.{suffix}', 'port': 8000 + suffix, 'valid': valid, 'country_code': country, } def test_get_webshare_ip_candidates_returns_empty_without_country(): service = ProxyService() assert service.get_webshare_ip_candidates(None) == [] def test_fetch_webshare_ip_list_sends_uppercase_country_code(monkeypatch): # Webshare's proxy-list *management* API wants uppercase country codes # (their own docs example: "FR,US") — the opposite convention from the # rotating-gateway username suffix, which wants lowercase. These are two # separate Webshare systems; do not "fix" them to match each other. service = ProxyService() captured = {} def fake_get(url, headers=None, params=None, timeout=None): captured['params'] = params return _FakeResponse({'results': [_fake_ip(1)], 'next': None}) monkeypatch.setattr(proxy_service_module.requests, 'get', fake_get) service.get_webshare_ip_candidates('gb') assert captured['params']['country_code__in'] == 'GB' def test_get_webshare_ip_candidates_filters_valid_and_respects_limit(monkeypatch): service = ProxyService() ips = [_fake_ip(1), _fake_ip(2, valid=False), _fake_ip(3), _fake_ip(4), _fake_ip(5)] monkeypatch.setattr( proxy_service_module.requests, 'get', lambda *a, **k: _FakeResponse({'results': ips, 'next': None}) ) candidates = service.get_webshare_ip_candidates('GB', limit=2) assert len(candidates) == 2 # The invalid one (suffix 2) must never appear, since it's filtered # before the random.sample() call — every candidate has real fields. for c in candidates: assert 'password' in c and c['password'] != 'pass2' def test_get_webshare_ip_candidates_dict_shape_uses_per_ip_credentials(monkeypatch): # Each Webshare list entry carries its OWN username/password/port — # genuinely different from _webshare_playwright()'s single account-level # rotating-gateway credentials at p.webshare.io:80. service = ProxyService() monkeypatch.setattr( proxy_service_module.requests, 'get', lambda *a, **k: _FakeResponse({'results': [_fake_ip(7)], 'next': None}) ) candidates = service.get_webshare_ip_candidates('GB', limit=1) assert candidates == [{'server': 'http://1.2.3.7:8007', 'username': 'user7', 'password': 'pass7'}] def test_get_webshare_ip_candidates_returns_empty_on_api_failure(monkeypatch): service = ProxyService() def raising_get(*a, **k): raise proxy_service_module.requests.RequestException('connection refused') monkeypatch.setattr(proxy_service_module.requests, 'get', raising_get) assert service.get_webshare_ip_candidates('GB') == [] def test_get_webshare_ip_candidates_returns_empty_when_no_results_for_country(monkeypatch): service = ProxyService() monkeypatch.setattr( proxy_service_module.requests, 'get', lambda *a, **k: _FakeResponse({'results': [], 'next': None}) ) assert service.get_webshare_ip_candidates('GB') == [] def test_fetch_webshare_ip_list_caches_within_ttl(monkeypatch): service = ProxyService() call_count = {'n': 0} def fake_get(*a, **k): call_count['n'] += 1 return _FakeResponse({'results': [_fake_ip(1)], 'next': None}) monkeypatch.setattr(proxy_service_module.requests, 'get', fake_get) monkeypatch.setattr(proxy_service_module.time, 'time', lambda: 1000.0) service.get_webshare_ip_candidates('GB') service.get_webshare_ip_candidates('GB') assert call_count['n'] == 1 # second call served from cache def test_fetch_webshare_ip_list_refetches_after_ttl_expires(monkeypatch): service = ProxyService() call_count = {'n': 0} current_time = {'t': 1000.0} def fake_get(*a, **k): call_count['n'] += 1 return _FakeResponse({'results': [_fake_ip(1)], 'next': None}) monkeypatch.setattr(proxy_service_module.requests, 'get', fake_get) monkeypatch.setattr(proxy_service_module.time, 'time', lambda: current_time['t']) service.get_webshare_ip_candidates('GB') current_time['t'] += proxy_service_module.WEBSHARE_IP_CACHE_TTL_SECONDS + 1 service.get_webshare_ip_candidates('GB') assert call_count['n'] == 2 def test_fetch_webshare_ip_list_negative_cache_uses_shorter_ttl(monkeypatch): service = ProxyService() call_count = {'n': 0} current_time = {'t': 1000.0} def raising_get(*a, **k): call_count['n'] += 1 raise proxy_service_module.requests.RequestException('down') monkeypatch.setattr(proxy_service_module.requests, 'get', raising_get) monkeypatch.setattr(proxy_service_module.time, 'time', lambda: current_time['t']) service.get_webshare_ip_candidates('GB') current_time['t'] += proxy_service_module.WEBSHARE_IP_CACHE_NEGATIVE_TTL_SECONDS + 1 service.get_webshare_ip_candidates('GB') assert call_count['n'] == 2 # negative cache expired, retried