From 6a259a501280cfdd32ceba480d819804aaf4726a Mon Sep 17 00:00:00 2001 From: JasonFraser Date: Wed, 1 Jul 2026 12:44:35 -0400 Subject: [PATCH] Phase 2: refactor legacy app.py into core/industries/config structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements CLAUDE.md Build Order Phase 2 — same scraping/extraction logic as the legacy app.py, reorganized into the three-layer structure and upgraded per §10's explicit instructions: - core/scraper.py — playwright-stealth replaces the manual add_init_script stealth block; smart proxy (Webshare→Bright Data) wired via core/proxy_service.py with block-detection + auto-escalation; check_robots_txt() gate added per §30 - core/extractor.py — OpenRouter model cycling, merging the legacy JSON-repair fallback chain with the spec's env-driven free/paid model list and Sentry breadcrumb on paid-fallback use - core/ai_fetch.py — AI web fetch fallback, ported unchanged - industries/adventure_travel/{fields,prompt}.py ported unchanged; schema.py added as the future V2 custom-fields attachment point - config/client.py — gspread Sheet write-back, ported unchanged - services/analysis_service.py now sources CACHE_STALENESS_HOURS from core/scraper.py, its canonical location per §7 Dropped the legacy's commented-out Groq/Gemini alternates — OpenRouter is the sole active provider and the dead code added nothing. requirements.txt: relaxed playwright to >=1.55.0 and pinned playwright-stealth==1.0.6 + setuptools<81 — Python 3.14 has no prebuilt wheels for the originally-listed pins, and playwright-stealth 2.x replaced the stealth_async(page) API §10 documents verbatim with a class-based one. 192 tests, 94% coverage. Also fixes a real bug surfaced by this phase: several test helpers mutated core.extractor / industries.adventure_travel.prompt via raw setattr instead of monkeypatch.setattr, silently corrupting shared module state across tests once those modules became real. Co-Authored-By: Claude Sonnet 5 --- backend/config/__init__.py | 0 backend/config/client.py | 62 ++++ backend/core/__init__.py | 0 backend/core/ai_fetch.py | 87 ++++++ backend/core/extractor.py | 129 ++++++++ backend/core/proxy_service.py | 103 ++++++ backend/core/scraper.py | 293 ++++++++++++++++++ backend/industries/__init__.py | 0 .../industries/adventure_travel/__init__.py | 0 backend/industries/adventure_travel/fields.py | 27 ++ backend/industries/adventure_travel/prompt.py | 116 +++++++ backend/industries/adventure_travel/schema.py | 44 +++ backend/requirements.txt | 13 +- backend/services/analysis_service.py | 9 +- backend/tests/core/__init__.py | 0 backend/tests/core/test_ai_fetch.py | 76 +++++ backend/tests/core/test_extractor.py | 132 ++++++++ backend/tests/core/test_proxy_service.py | 87 ++++++ backend/tests/core/test_scraper.py | 192 ++++++++++++ .../tests/services/test_analysis_service.py | 22 +- .../tests/services/test_battlecard_service.py | 5 +- backend/tests/test_api_extended.py | 9 +- backend/tests/test_config_client.py | 76 +++++ .../tests/test_industries_adventure_travel.py | 64 ++++ 24 files changed, 1533 insertions(+), 13 deletions(-) create mode 100644 backend/config/__init__.py create mode 100644 backend/config/client.py create mode 100644 backend/core/__init__.py create mode 100644 backend/core/ai_fetch.py create mode 100644 backend/core/extractor.py create mode 100644 backend/core/proxy_service.py create mode 100644 backend/core/scraper.py create mode 100644 backend/industries/__init__.py create mode 100644 backend/industries/adventure_travel/__init__.py create mode 100644 backend/industries/adventure_travel/fields.py create mode 100644 backend/industries/adventure_travel/prompt.py create mode 100644 backend/industries/adventure_travel/schema.py create mode 100644 backend/tests/core/__init__.py create mode 100644 backend/tests/core/test_ai_fetch.py create mode 100644 backend/tests/core/test_extractor.py create mode 100644 backend/tests/core/test_proxy_service.py create mode 100644 backend/tests/core/test_scraper.py create mode 100644 backend/tests/test_config_client.py create mode 100644 backend/tests/test_industries_adventure_travel.py diff --git a/backend/config/__init__.py b/backend/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/config/client.py b/backend/config/client.py new file mode 100644 index 0000000..febe793 --- /dev/null +++ b/backend/config/client.py @@ -0,0 +1,62 @@ +import os +import json +import gspread +from google.oauth2.service_account import Credentials + +# Client-specific Sheet configuration — swapped per client within the +# adventure_travel vertical. Ported unchanged from the legacy app.py. +SHEET_ID = os.getenv('SHEET_ID') +STANDARD_TAB = os.getenv('STANDARD_TAB', 'Standard Competitive Tab') +NGS_TAB = os.getenv('NGS_TAB', 'NGS Competitive Tab') +SERVICE_ACCOUNT_JSON = os.getenv('GOOGLE_SERVICE_ACCOUNT_JSON') + + +def get_sheet_client(): + """Authorizes a gspread client against the Google service account credentials.""" + scopes = [ + 'https://www.googleapis.com/auth/spreadsheets', + 'https://www.googleapis.com/auth/drive', + ] + info = json.loads(SERVICE_ACCOUNT_JSON) + creds = Credentials.from_service_account_info(info, scopes=scopes) + return gspread.authorize(creds) + + +def write_to_sheet(tab_name, col_index, data, fields): + """ + Writes one competitor's extracted fields into a single column of + the client Sheet, using the field→row map for this tab type + (STANDARD_FIELDS or NGS_FIELDS). + + Data flow: + tab_name + col_index + extracted data dict + field row map → + batch_update() with one cell write per non-empty field → + Sheet updated in a single API call + """ + client = get_sheet_client() + sheet = client.open_by_key(SHEET_ID) + ws = sheet.worksheet(tab_name) + updates = [] + for field, row in fields.items(): + value = data.get(field) + if value is None or value == '': + continue + updates.append({ + 'range': gspread.utils.rowcol_to_a1(row, col_index), + 'values': [[value]], + }) + if updates: + ws.batch_update(updates) + + +def get_competitors_from_db(): + """Reads the client's 'Competitor DB' tab and returns only the active rows.""" + client = get_sheet_client() + sheet = client.open_by_key(SHEET_ID) + ws = sheet.worksheet('Competitor DB') + rows = ws.get_all_records() + return [ + {'style': r['Travel Style'], 'name': r['Competitor Name'], 'url': r['Website URL'], 'active': r['Active']} + for r in rows + if str(r.get('Active', '')).strip().lower() == 'yes' + ] diff --git a/backend/core/__init__.py b/backend/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/core/ai_fetch.py b/backend/core/ai_fetch.py new file mode 100644 index 0000000..f70bec8 --- /dev/null +++ b/backend/core/ai_fetch.py @@ -0,0 +1,87 @@ +import os +import logging +from urllib.parse import urlparse +from openai import OpenAI + +logger = logging.getLogger(__name__) + +OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY') + +# Note: this fallback calls OpenRouter's server-side web_search tool +# (Exa engine) rather than making a direct HTTP request to the +# competitor's domain — the search itself runs on OpenRouter's +# infrastructure, not ours, so core/proxy_service.py's Webshare/Bright +# Data routing has no attachment point here. Ported unchanged from the +# legacy app.py per CLAUDE.md §10 ("logic stays identical"). + + +def fetch_with_ai(url): + """ + AI web fetch fallback for bot-protected sites. Fires only when + Playwright returns under 1000 chars. Uses OpenRouter's web_search + server tool with the Exa engine, restricted to the competitor's + own domain. Cost: ~$0.02/call + tokens — only fires for + bot-protected sites, not every scrape. + + Data flow: + url → domain extracted → OpenRouter chat completion with + web_search tool (allowed_domains=[domain]) → + response text + any citation content → + { text, tables: '', url, char_count, success } or None if the + fallback also returns too little content + """ + logger.info(f'Playwright failed — attempting AI web fetch for {url}') + try: + client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1') + domain = urlparse(url).netloc + + response = client.chat.completions.create( + model='openai/gpt-4o-mini', + messages=[{ + 'role': 'user', + 'content': ( + 'I am a travel industry analyst conducting competitive research. ' + 'Please search for this tour page and extract the following factual data points: ' + 'trip name, trip code, price, duration in days, maximum group size, ' + 'total included meals, hotel names, departure dates, start location, ' + 'end location, service level, and key activities. ' + f'Return all factual information you find.\n\nURL: {url}' + ) + }], + tools=[{ + 'type': 'openrouter:web_search', + 'parameters': { + 'engine': 'exa', + 'max_results': 5, + 'search_context_size': 'high', + 'allowed_domains': [domain], + } + }], + temperature=0.0, + max_tokens=8192 + ) + + msg = response.choices[0].message + text = msg.content if isinstance(msg.content, str) else '' + + # Also pull content from tool-result annotations if present + if hasattr(msg, 'annotations') and msg.annotations: + for annotation in msg.annotations: + if hasattr(annotation, 'url_citation') and annotation.url_citation: + citation_content = getattr(annotation.url_citation, 'content', '') + if citation_content: + text = text + '\n\n' + citation_content + + text = text.strip() + char_count = len(text) + logger.info(f'AI web fetch returned {char_count} chars for {url}') + + if char_count > 300: + return {'text': text[:25000], 'tables': '', 'url': url, 'char_count': char_count, 'success': True} + + logger.warning(f'AI web fetch also returned low content for {url}') + return None + + except Exception as e: + logger.error(f'AI web fetch failed for {url}: {str(e)}') + return None diff --git a/backend/core/extractor.py b/backend/core/extractor.py new file mode 100644 index 0000000..ee07b61 --- /dev/null +++ b/backend/core/extractor.py @@ -0,0 +1,129 @@ +import os +import re +import json +import logging +from openai import OpenAI +import sentry_sdk + +logger = logging.getLogger(__name__) + +OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY') +FREE_MODELS = [m.strip() for m in os.getenv('OPENROUTER_FREE_MODELS', '').split(',') if m.strip()] +FALLBACK_MODEL = os.getenv('OPENROUTER_FALLBACK_MODEL', 'anthropic/claude-haiku-4-5') + + +class ExtractorError(Exception): + """Raised when every OpenRouter model (free tier + paid fallback) has failed.""" + pass + + +class RateLimitError(Exception): + """Raised internally when an OpenRouter model returns a 429 / rate-limit response.""" + pass + + +def _parse_json_response(text, model, label): + """ + Parses a model's raw text response into JSON via progressively more + aggressive repairs — ported unchanged from the legacy app.py + extraction logic: raw parse → regex object extraction → + trailing-comma/unquoted-key repair. + + Data flow: + raw model text → strip code fences → + json.loads() (fast path) → + regex-extract the outermost {...} block and retry → + strip trailing commas + quote bare keys and retry → + parsed dict returned, or ValueError if all three fail + """ + cleaned = re.sub(r'```json|```', '', text).strip() + + try: + return json.loads(cleaned) + except json.JSONDecodeError: + pass + + match = re.search(r'\{[\s\S]*\}', cleaned) + if match: + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + pass + + try: + fixed = re.sub(r',\s*}', '}', cleaned) + fixed = re.sub(r',\s*]', ']', fixed) + fixed = re.sub(r'([{,])\s*([a-zA-Z0-9_]+)\s*:', r'\1"\2":', fixed) + return json.loads(fixed) + except json.JSONDecodeError: + raise ValueError(f'Could not parse response from {model} for {label}') + + +def _call_openrouter(model, prompt, content): + """ + Issues one chat completion request to OpenRouter for `model`. + `content` is appended to `prompt` when present — most callers + (via industries.adventure_travel.prompt.build_prompt) already embed + page content directly into `prompt`, so `content` is typically + empty; analysis_service's refresh path passes it explicitly too, + which is harmless (the model just sees the page text once via the + prompt and, redundantly, once more here). + + Data flow: + model + prompt + content → OpenAI client (OpenRouter base_url) → + chat.completions.create() → raise RateLimitError on 429/rate-limit + responses (caller moves to the next model) → raw text → + _parse_json_response() → parsed dict returned + """ + client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1') + message = f'{prompt}\n\n{content}' if content else prompt + + try: + response = client.chat.completions.create( + model=model, + messages=[{'role': 'user', 'content': message}], + temperature=0.1, + max_tokens=4096 + ) + except Exception as e: + error_str = str(e) + if '429' in error_str or 'rate_limit' in error_str.lower(): + raise RateLimitError(error_str) + raise + + text = response.choices[0].message.content + return _parse_json_response(text, model, 'extraction') + + +def extract_with_openrouter(prompt, content=''): + """ + Cycles through free OpenRouter models in sequence. Falls back to + the paid fallback model if all free models fail or rate-limit. + Never returns None — raises ExtractorError only after every model + (free + paid) has failed. + + Data flow: + prompt + content → for each model in FREE_MODELS + [FALLBACK_MODEL]: + _call_openrouter() → success: return parsed JSON + (logging + a Sentry breadcrumb if the paid fallback had to be used) → + RateLimitError: log, try next model → + any other error: log, try next model → + all models exhausted → raise ExtractorError + """ + models_to_try = FREE_MODELS + [FALLBACK_MODEL] + + for model in models_to_try: + try: + result = _call_openrouter(model, prompt, content) + if model == FALLBACK_MODEL and FREE_MODELS: + logger.warning(f'Free models exhausted — used paid fallback: {model}') + sentry_sdk.capture_message('OpenRouter paid fallback used', level='warning') + return result + except RateLimitError: + logger.warning(f'Rate limit on {model} — trying next') + continue + except Exception as e: + logger.error(f'Model {model} failed: {str(e)}') + continue + + raise ExtractorError('All OpenRouter models exhausted including paid fallback') diff --git a/backend/core/proxy_service.py b/backend/core/proxy_service.py new file mode 100644 index 0000000..2cc2a6b --- /dev/null +++ b/backend/core/proxy_service.py @@ -0,0 +1,103 @@ +import os +from datetime import datetime, timezone +from repositories.proxy_repository import proxy_repository + +WEBSHARE_USER = os.getenv('WEBSHARE_USER') +WEBSHARE_PASS = os.getenv('WEBSHARE_PASS') +BRIGHTDATA_USER = os.getenv('BRIGHTDATA_USER') +BRIGHTDATA_PASS = os.getenv('BRIGHTDATA_PASS') + + +class ProxyService: + """ + Smart proxy provider — uses Webshare by default with country + targeting, auto-escalates to Bright Data for domains that block + Webshare. This is the ONLY place in the codebase that knows about + any proxy provider — swapping providers or switching logic means + changing only this file (CLAUDE.md §10). + + Country targeting uses competitors.primary_market (ISO code e.g. + GB, US, AU) passed through from the competitor record — no + hardcoded country map in code. + + Data flow (first request to a domain): + domain + country → proxy_overrides lookup → no override found → + Webshare config with country targeting returned → + scraper attempts request → if blocked → + escalate_to_brightdata(domain) called → + Bright Data config returned → scraper retries → + override stored in PocketBase for this domain + + Data flow (subsequent requests to a known blocked domain): + domain + country → proxy_overrides lookup → brightdata override + found → Bright Data config returned immediately, no Webshare attempt + """ + + def get_proxy_for_domain(self, domain, country=None): + """Returns a `requests`-style proxy config, routed to Bright Data if previously blocked.""" + override = proxy_repository.get_by_domain(domain) + if override and override.get('provider') == 'brightdata': + return self._brightdata_requests() + return self._webshare_requests(country=country) + + def get_playwright_proxy_for_domain(self, domain, country=None): + """Returns a Playwright proxy config, routed to Bright Data if previously blocked.""" + override = proxy_repository.get_by_domain(domain) + if override and override.get('provider') == 'brightdata': + return self._brightdata_playwright() + return self._webshare_playwright(country=country) + + def escalate_to_brightdata(self, domain): + """ + Records that Webshare was blocked on this domain. Upserts a + proxy_override so future requests skip Webshare entirely for + this domain. + """ + existing = proxy_repository.get_by_domain(domain) + if existing: + proxy_repository.update(existing['id'], { + 'provider': 'brightdata', + 'blocked_count': existing.get('blocked_count', 0) + 1, + 'last_blocked': datetime.now(timezone.utc).isoformat(), + }) + else: + proxy_repository.create({ + 'domain': domain, + 'provider': 'brightdata', + 'blocked_count': 1, + 'last_blocked': datetime.now(timezone.utc).isoformat(), + }) + + def _webshare_requests(self, country=None): + """Webshare `requests` proxy config. Format: username-{COUNTRY}-rotate. No country = full pool rotation.""" + suffix = f'-{country.upper()}-rotate' if country else '-rotate' + username = f'{WEBSHARE_USER}{suffix}' + return { + 'http': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80', + 'https': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80', + } + + def _webshare_playwright(self, country=None): + """Webshare Playwright proxy config with optional country targeting.""" + suffix = f'-{country.upper()}-rotate' if country else '-rotate' + return { + 'server': 'http://p.webshare.io:80', + 'username': f'{WEBSHARE_USER}{suffix}', + 'password': WEBSHARE_PASS, + } + + def _brightdata_requests(self): + return { + 'http': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:22225', + 'https': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:22225', + } + + def _brightdata_playwright(self): + return { + 'server': 'http://brd.superproxy.io:22225', + 'username': BRIGHTDATA_USER, + 'password': BRIGHTDATA_PASS, + } + + +proxy_service = ProxyService() diff --git a/backend/core/scraper.py b/backend/core/scraper.py new file mode 100644 index 0000000..05de2a6 --- /dev/null +++ b/backend/core/scraper.py @@ -0,0 +1,293 @@ +import re +import asyncio +import logging +from urllib.parse import urlparse, urlencode, parse_qs, urlunparse +from urllib.robotparser import RobotFileParser + +from playwright.async_api import async_playwright +from playwright_stealth import stealth_async +from bs4 import BeautifulSoup + +from core.proxy_service import proxy_service + +logger = logging.getLogger(__name__) + +# Default staleness threshold for the on-demand fast/refresh path +# decision (CLAUDE.md §7). This is the canonical, adjustable location +# per §7 — services/analysis_service.py imports it from here rather +# than redefining it. +CACHE_STALENESS_HOURS = 24 + +# Block only known third-party analytics/ad domains — never block the +# competitor's own first-party assets. +BLOCKED_DOMAINS = [ + 'google-analytics.com', 'googletagmanager.com', 'doubleclick.net', + 'facebook.net', 'facebook.com/tr', 'hotjar.com', 'intercom.io', + 'zendesk.com', 'hubspot.com', 'marketo.com', 'segment.com', + 'amplitude.com', 'mixpanel.com', 'optimizely.com', 'crazyegg.com', +] + + +def extract_domain(url): + """Strips scheme/path down to a bare domain — used for proxy_overrides and robots.txt lookups.""" + return urlparse(url if '://' in url else f'//{url}').netloc.lower().lstrip('www.') + + +def check_robots_txt(url): + """ + Checks if the given URL is allowed to be scraped per robots.txt. + Returns True if scraping is permitted, False if disallowed. + Defaults to True if robots.txt cannot be fetched (permissive + default) — CLAUDE.md §30 Web Scraping Legal Policy. + + Data flow: + url → extract root domain → fetch /robots.txt → + parse with urllib.robotparser → + check if Bettersight's user-agent is allowed for this path + """ + root = f'{urlparse(url).scheme}://{urlparse(url).netloc}' + parser = RobotFileParser() + parser.set_url(f'{root}/robots.txt') + try: + parser.read() + return parser.can_fetch('*', url) + except Exception: + return True + + +def clean_url(url): + """Strips URL fragments and common tracking parameters.""" + parsed = urlparse(url) + strip_params = { + 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', + 'referrerView', 'ref', 'referrer', 'source', 'fbclid', 'gclid', + 'mc_cid', 'mc_eid', '_ga', 'affiliate', + } + if parsed.query: + params = parse_qs(parsed.query, keep_blank_values=True) + cleaned = {k: v for k, v in params.items() if k not in strip_params} + query = urlencode(cleaned, doseq=True) + else: + query = '' + clean = urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, query, '')) + if clean != url: + logger.info(f'URL cleaned: {url} → {clean}') + return clean + + +def _is_blocked(result): + """ + Detects common block signals from a scrape result. Returns True if + the response looks like a bot wall rather than real content. + Signals: low char count, HTTP 403/429, Cloudflare challenge, CAPTCHA text. + """ + if result.get('char_count', 0) < 500: + return True + if result.get('status_code') in (403, 429): + return True + block_signals = ['cf-browser-verification', 'captcha', 'access denied', 'blocked', 'robot'] + return any(signal in result.get('text', '').lower() for signal in block_signals) + + +async def _render_with_proxy(url, proxy): + """ + Launches a stealth Playwright browser through the given proxy + config and extracts cleaned page text + tables. This is the actual + page-rendering logic — split out from render_page() so it can be + retried with a different proxy (Bright Data) on the first attempt + being blocked, without duplicating the browser setup. + + Data flow: + url + proxy config → Playwright chromium launch (proxied) → + playwright-stealth applied → navigate → dismiss cookie banner → + wait for content → scroll to trigger lazy-loading → + HTML → BeautifulSoup cleaning → { text, tables, url, char_count, success } + """ + async with async_playwright() as p: + browser = await p.chromium.launch( + headless=True, + proxy=proxy, + args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] + ) + context = await browser.new_context( + user_agent=( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' + ), + viewport={'width': 1366, 'height': 768}, + locale='en-GB', + timezone_id='Europe/London', + extra_http_headers={ + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-GB,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'DNT': '1', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + } + ) + page = await context.new_page() + + # Comprehensive stealth (navigator.webdriver, plugins, canvas/WebGL + # fingerprint, audio context, chrome runtime, ~20 evasions total) + # replaces the old manual add_init_script block per CLAUDE.md §10. + await stealth_async(page) + + async def block_trackers(route): + if any(domain in route.request.url for domain in BLOCKED_DOMAINS): + await route.abort() + else: + await route.continue_() + + await page.route('**/*', block_trackers) + + try: + try: + await page.goto(url, wait_until='domcontentloaded', timeout=30000) + logger.info(f'Loaded: {url}') + except Exception as e: + logger.warning(f'domcontentloaded failed, trying networkidle: {str(e)[:100]}') + await page.goto(url, wait_until='networkidle', timeout=60000) + + # Dismiss cookie consent banners + try: + await page.click( + 'button:has-text("Accept All"), button:has-text("Accept all"), ' + 'button:has-text("Accept"), button:has-text("OK"), ' + 'button:has-text("I agree"), button:has-text("Agree"), ' + '[id*="cookie"] button, [class*="cookie"] button, ' + '[id*="consent"] button, [class*="consent"] button', + timeout=3000 + ) + logger.info('Cookie banner dismissed') + await page.wait_for_timeout(1000) + except Exception: + pass + + # Wait for meaningful content + try: + await page.wait_for_selector( + 'h1, .trip-overview, main, article, #overview, ' + '[class*="itinerary"], [class*="price"], [class*="trip"]', + timeout=8000 + ) + except Exception: + pass + + await page.wait_for_timeout(3000) + + # Two scroll passes to trigger lazy-loaded content + for _ in range(2): + await page.evaluate('window.scrollTo(0, document.body.scrollHeight)') + await page.wait_for_timeout(1500) + await page.evaluate('window.scrollTo(0, 0)') + await page.wait_for_timeout(500) + + await page.evaluate('window.scrollTo(0, document.body.scrollHeight)') + await page.wait_for_timeout(2000) + + html = await page.content() + soup = BeautifulSoup(html, 'lxml') + + for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'iframe', 'noscript', 'svg', 'button', 'form']): + tag.decompose() + + main_content = ( + soup.find('main') or soup.find('article') or soup.find(id='main') + or soup.find(id='content') or soup.find(class_='content') or soup + ) + + text = main_content.get_text(separator=' ', strip=True) + text = re.sub(r'\s+', ' ', text).strip() + + tables = [t.get_text(separator=' | ', strip=True) for t in soup.find_all('table')] + table_text = '\n'.join(tables) + char_count = len(text) + + if char_count < 500: + logger.warning(f'Low char count ({char_count}) for {url} — will attempt AI web fetch fallback') + + return { + 'text': text[:25000], + 'tables': table_text[:5000], + 'url': url, + 'char_count': char_count, + 'success': True, + } + + except Exception as e: + logger.error(f'Render failed for {url}: {str(e)}') + return {'text': f'Render failed: {str(e)}', 'tables': '', 'url': url, 'char_count': 0, 'success': False} + finally: + await browser.close() + + +async def render_page(url, country=None): + """ + Renders a page using the smart proxy service with country + targeting. `country` comes from competitors.primary_market in + PocketBase. Detects blocks and auto-escalates to Bright Data if + Webshare fails. + + Data flow: + url + country → domain extracted → + proxy_service.get_playwright_proxy_for_domain(domain, country) → + _render_with_proxy() → if blocked → escalate_to_brightdata(domain) → + retry once with Bright Data (no country targeting at fallback level) + """ + domain = extract_domain(url) + proxy = proxy_service.get_playwright_proxy_for_domain(domain, country) + result = await _render_with_proxy(url, proxy) + + if _is_blocked(result): + logger.warning(f'Webshare blocked on {domain} — escalating to Bright Data') + proxy_service.escalate_to_brightdata(domain) + result = await _render_with_proxy(url, proxy_service._brightdata_playwright()) + + return result + + +def scrape(url, country=None): + """ + Synchronous entry point for scraping one competitor page. Checks + robots.txt first (legal/compliance gate per §30), then renders via + Playwright, falling back to the AI web fetch path if Playwright's + char count is too low (bot-protected sites). + + Data flow: + url + country → clean_url() → check_robots_txt() → + disallowed: return failure immediately (no scrape attempted) → + allowed: render_page() (sync wrapper via nest_asyncio) → + char_count < 1000: fetch_with_ai() fallback → + final { text, tables, url, char_count, success } returned + """ + import nest_asyncio + nest_asyncio.apply() + + url = clean_url(url) + + if not check_robots_txt(url): + logger.warning(f'robots.txt disallows scraping {url} — skipping') + return { + 'text': 'Scraping disallowed by robots.txt for this URL', + 'tables': '', 'url': url, 'char_count': 0, 'success': False, + } + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete(render_page(url, country=country)) + finally: + loop.close() + + if result.get('char_count', 0) < 1000 and result.get('success'): + logger.info(f'Playwright char count low — trying AI web fetch for {url}') + from core.ai_fetch import fetch_with_ai + ai_result = fetch_with_ai(url) + if ai_result: + return ai_result + + return result diff --git a/backend/industries/__init__.py b/backend/industries/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/industries/adventure_travel/__init__.py b/backend/industries/adventure_travel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/industries/adventure_travel/fields.py b/backend/industries/adventure_travel/fields.py new file mode 100644 index 0000000..d31a275 --- /dev/null +++ b/backend/industries/adventure_travel/fields.py @@ -0,0 +1,27 @@ +# Column position maps for the legacy Google Sheet write-back +# (config/client.py). Ported unchanged from the pre-refactor app.py — +# these positions are specific to the existing client Sheet template +# and must not be renumbered without also updating that template. + +STANDARD_FIELDS = { + 'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10, + 'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14, + 'startLocation': 15, 'endLocation': 16, 'departures': 17, + 'seasonality': 19, 'startDays': 20, 'targetAudience': 21, + 'hotels': 22, 'activities': 23, 'relevancy': 24, + 'majorityPrice': 25, 'priceLow': 26, 'priceHigh': 27, + 'comments': 30, 'dateUsed': 31, + 'audPrice': 32, 'cadPrice': 33, 'eurPrice': 34, 'gbpPrice': 35, +} + +NGS_FIELDS = { + 'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10, + 'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14, + 'startLocation': 15, 'endLocation': 16, 'departures': 17, + 'seasonality': 19, 'startDays': 20, 'targetAudience': 21, + 'hotels': 22, 'activities': 23, 'exclusiveAccess': 24, + 'groupLeader': 25, 'sustainability': 26, 'relevancy': 27, + 'majorityPrice': 28, 'priceLow': 29, 'priceHigh': 30, + 'comments': 33, 'dateUsed': 34, + 'audPrice': 35, 'cadPrice': 36, 'eurPrice': 37, 'gbpPrice': 38, +} diff --git a/backend/industries/adventure_travel/prompt.py b/backend/industries/adventure_travel/prompt.py new file mode 100644 index 0000000..4980a85 --- /dev/null +++ b/backend/industries/adventure_travel/prompt.py @@ -0,0 +1,116 @@ +# Prompt template and injection logic for the adventure travel +# vertical — ported unchanged from the legacy app.py. Per CLAUDE.md +# §10, pivoting to a new industry means adding a new folder under +# industries/ with these same three files; core/ is never touched. + + +def build_prompt(competitor_name, page_data, product_name, + destination, duration, travel_style, is_ngs=False): + """ + Builds the full extraction prompt for one competitor page, embedding + the scraped page content directly into the returned template. + + Data flow: + competitor_name + page_data (scraped text/tables/url) + client + product context (name/destination/duration/style) + is_ngs flag → + NGS-specific field block appended if is_ngs → + full prompt string returned, ready to send to core.extractor + """ + ngs_fields = ''' + "exclusiveAccess": "exclusive access or special unique inclusions", + "groupLeader": "group leader and/or local expert description", + "sustainability": "sustainability inclusions or null",''' if is_ngs else '' + + content = page_data['text'] if page_data else '[PAGE CONTENT WILL BE INSERTED HERE]' + if page_data and page_data.get('tables'): + content += f'\n\nTABLE DATA:\n{page_data["tables"]}' + + url = page_data['url'] if page_data else '[COMPETITOR URL]' + + return f'''You are a travel industry analyst helping G Adventures research competitors. +Be concise — use short phrases not full sentences for all fields except comments. +Read the ENTIRE page content carefully before extracting any fields. +Pay particular attention to itinerary sections, departure tables, and pricing grids +which may appear later in the content. + +The content below was scraped from the {competitor_name} tour page at: {url} + +This is the specific page selected as most comparable to: +- G Adventures Product: {product_name} +- Destination: {destination} +- Duration: ~{duration} days +- G Adventures Travel Style: {travel_style} + +PAGE CONTENT: +--- +{content} +--- + +CRITICAL PRICING RULES: +- majorityPrice: Standard/regular adult price for the most common departure. + IGNORE sale, promotional, early bird, or "was/now" pricing. Use non-discounted rack rate only. +- Prices may appear as "from $X", "per person from $X", or inside a departures table or pricing grid. + Extract the most commonly listed standard adult price regardless of how it is labelled on the page. +- priceLow: Lowest standard price across all departures shown (not sale price). +- priceHigh: Highest standard price across all departures shown (not sale price). +- If a site requires clicking "see more departures" to show all dates, note this in seasonality + and use only prices visible on the main listing page. +- Multi-currency: only fill if the price is explicitly listed in that currency on the page. + Do not convert. If the page shows GBP only, fill gbpPrice and leave USD fields null. + +MEALS RULE: +- meals: Return the TOTAL number of included meals as a single integer only. No text description. +- Meals may be listed individually per day in the itinerary section + (e.g. "Day 1: Breakfast", "Day 3: Breakfast and Dinner"). + Count every individual meal mention across all itinerary days. + Do not rely on a summary box — read through the full itinerary to count. +- Count each meal type separately: 1 breakfast + 1 lunch + 1 dinner = 3 meals. + +Return ONLY a valid JSON object. No markdown, no code fences, no explanation. + +{{ + "company": "{competitor_name}", + "link": "{url}", + "tripName": "exact trip name from the page", + "tripCode": "trip code if shown, otherwise null", + "serviceLevel": "accommodation service level", + "groupSize": max group size as integer, + "duration": number of days as integer, + "meals": total included meals as integer,{ngs_fields} + "startLocation": "departure city", + "endLocation": "end city", + "departures": approximate departures per year as integer or null, + "seasonality": "price bands by season. Note if more departures are hidden behind a button.", + "startDays": "departure days of week", + "targetAudience": "target traveller", + "hotels": "hotel types or properties", + "activities": "key included activities", + "relevancy": relevancy score 1-5 as integer, + "majorityPrice": standard adult USD price as number or null, + "priceLow": lowest standard USD price as number or null, + "priceHigh": highest standard USD price as number or null, + "comments": "analysis: key differences from G Adventures, strengths, weaknesses, pricing position", + "dateUsed": "specific departure date used for majority price", + "audPrice": standard AUD price as number or null, + "cadPrice": standard CAD price as number or null, + "eurPrice": standard EUR price as number or null, + "gbpPrice": standard GBP price as number or null +}}''' + + +def inject_page_content(prompt_template, competitor_name, page_data): + """ + Injects real scraped content into a user-edited prompt template + (used when the PM has customised the prompt via the dashboard's + prompt preview/edit flow instead of using build_prompt()'s default). + """ + content = page_data['text'] + if page_data.get('tables'): + content += f'\n\nTABLE DATA:\n{page_data["tables"]}' + result = prompt_template.replace('[PAGE CONTENT WILL BE INSERTED HERE]', content) + result = result.replace('[COMPETITOR URL]', page_data['url']) + result = result.replace( + competitor_name + ' tour page at: [COMPETITOR URL]', + competitor_name + f' tour page at: {page_data["url"]}' + ) + return result diff --git a/backend/industries/adventure_travel/schema.py b/backend/industries/adventure_travel/schema.py new file mode 100644 index 0000000..ded36f2 --- /dev/null +++ b/backend/industries/adventure_travel/schema.py @@ -0,0 +1,44 @@ +# JSON extraction schema for the adventure travel vertical, expressed +# as data rather than embedded in the prompt template string. This +# mirrors the JSON shape at the end of prompt.build_prompt()'s output +# and exists as the attachment point for CLAUDE.md §28's Version 2 +# custom extraction fields (build_json_schema() appends tenant custom +# fields to whichever of these two dicts applies) — that build_json_schema() +# function itself is out of scope until Discover/Intelligence tier work +# begins; this file only establishes the base schema it would extend. + +STANDARD_SCHEMA = { + 'company': 'text — competitor name', + 'link': 'text — page URL', + 'tripName': 'text — exact trip name from the page', + 'tripCode': 'text — trip code if shown, otherwise null', + 'serviceLevel': 'text — accommodation service level', + 'groupSize': 'integer — max group size', + 'duration': 'integer — number of days', + 'meals': 'integer — total included meals', + 'startLocation': 'text — departure city', + 'endLocation': 'text — end city', + 'departures': 'integer — approximate departures per year, or null', + 'seasonality': 'text — price bands by season', + 'startDays': 'text — departure days of week', + 'targetAudience': 'text — target traveller', + 'hotels': 'text — hotel types or properties', + 'activities': 'text — key included activities', + 'relevancy': 'integer — relevancy score 1-5', + 'majorityPrice': 'price — standard adult USD price or null', + 'priceLow': 'price — lowest standard USD price or null', + 'priceHigh': 'price — highest standard USD price or null', + 'comments': 'text — analysis: differences, strengths, weaknesses, pricing position', + 'dateUsed': 'text — specific departure date used for majority price', + 'audPrice': 'price — standard AUD price or null', + 'cadPrice': 'price — standard CAD price or null', + 'eurPrice': 'price — standard EUR price or null', + 'gbpPrice': 'price — standard GBP price or null', +} + +NGS_SCHEMA = { + **STANDARD_SCHEMA, + 'exclusiveAccess': 'text — exclusive access or special unique inclusions', + 'groupLeader': 'text — group leader and/or local expert description', + 'sustainability': 'text — sustainability inclusions or null', +} diff --git a/backend/requirements.txt b/backend/requirements.txt index bad405c..4751a58 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -14,8 +14,19 @@ stripe==10.7.0 resend==2.4.0 # ── Phase 2 — scraping engine (moved from legacy app.py) ───── -playwright==1.46.0 +# playwright is unpinned to a minimum, not exact — 1.46.0's pinned +# greenlet dependency has no prebuilt wheel on newer Python versions +# and fails to build from source; newer playwright releases pull in a +# newer greenlet that does ship prebuilt wheels. +playwright>=1.55.0 +# playwright-stealth is pinned to 1.0.6 (not the 2.x line) because +# CLAUDE.md §10 documents the 1.x `stealth_async(page)` function API +# verbatim; playwright-stealth 2.x replaced it with a class-based +# `Stealth().apply_stealth_async(...)` API — a breaking change from +# what's specified. 1.0.6 depends on pkg_resources, which setuptools +# >=81 no longer ships — hence the setuptools pin below. playwright-stealth==1.0.6 +setuptools<81 beautifulsoup4==4.12.3 lxml==5.3.0 nest_asyncio==1.6.0 diff --git a/backend/services/analysis_service.py b/backend/services/analysis_service.py index 289f3b1..fb32679 100644 --- a/backend/services/analysis_service.py +++ b/backend/services/analysis_service.py @@ -5,9 +5,6 @@ from repositories.product_repository import product_repository logger = logging.getLogger(__name__) -# Default staleness threshold — configurable here only, per CLAUDE.md §7. -CACHE_STALENESS_HOURS = 24 - def needs_refresh(competitor): """ @@ -32,6 +29,12 @@ def needs_refresh(competitor): except (ValueError, AttributeError): return True + # CACHE_STALENESS_HOURS' canonical, adjustable home is core/scraper.py + # per CLAUDE.md §7 ("in core/scraper.py — adjust here only") — imported + # lazily here for the same reason every other core.* reference in this + # file is lazy: this module must stay importable independent of Phase 2. + from core.scraper import CACHE_STALENESS_HOURS + age = datetime.now(timezone.utc) - scraped_at return age > timedelta(hours=CACHE_STALENESS_HOURS) diff --git a/backend/tests/core/__init__.py b/backend/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/core/test_ai_fetch.py b/backend/tests/core/test_ai_fetch.py new file mode 100644 index 0000000..5453631 --- /dev/null +++ b/backend/tests/core/test_ai_fetch.py @@ -0,0 +1,76 @@ +from core import ai_fetch + + +class _FakeMessage: + def __init__(self, content, annotations=None): + self.content = content + self.annotations = annotations + + +class _FakeChoice: + def __init__(self, message): + self.message = message + + +class _FakeResponse: + def __init__(self, message): + self.choices = [_FakeChoice(message)] + + +def test_fetch_with_ai_returns_result_when_content_sufficient(monkeypatch): + class _FakeClient: + class chat: + class completions: + @staticmethod + def create(**kwargs): + return _FakeResponse(_FakeMessage('x' * 400)) + + monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient()) + + result = ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru') + + assert result['success'] is True + assert result['char_count'] == 400 + + +def test_fetch_with_ai_returns_none_when_content_too_short(monkeypatch): + class _FakeClient: + class chat: + class completions: + @staticmethod + def create(**kwargs): + return _FakeResponse(_FakeMessage('too short')) + + monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient()) + + assert ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru') is None + + +def test_fetch_with_ai_returns_none_on_exception(monkeypatch): + class _FakeClient: + class chat: + class completions: + @staticmethod + def create(**kwargs): + raise Exception('api down') + + monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient()) + + assert ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru') is None + + +def test_fetch_with_ai_appends_citation_content(monkeypatch): + citation = type('Citation', (), {'content': 'extra citation text'})() + annotation = type('Annotation', (), {'url_citation': citation})() + + class _FakeClient: + class chat: + class completions: + @staticmethod + def create(**kwargs): + return _FakeResponse(_FakeMessage('x' * 350, annotations=[annotation])) + + monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient()) + + result = ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru') + assert 'extra citation text' in result['text'] diff --git a/backend/tests/core/test_extractor.py b/backend/tests/core/test_extractor.py new file mode 100644 index 0000000..04aa122 --- /dev/null +++ b/backend/tests/core/test_extractor.py @@ -0,0 +1,132 @@ +import pytest +from core import extractor +from core.extractor import extract_with_openrouter, ExtractorError, RateLimitError, _parse_json_response + + +def test_parse_json_response_handles_clean_json(): + result = _parse_json_response('{"tripName": "Peru Classic"}', 'model-a', 'test') + assert result == {'tripName': 'Peru Classic'} + + +def test_parse_json_response_strips_code_fences(): + result = _parse_json_response('```json\n{"tripName": "Peru Classic"}\n```', 'model-a', 'test') + assert result == {'tripName': 'Peru Classic'} + + +def test_parse_json_response_extracts_object_from_surrounding_text(): + result = _parse_json_response('Here is the data: {"tripName": "Peru Classic"} Thanks!', 'model-a', 'test') + assert result == {'tripName': 'Peru Classic'} + + +def test_parse_json_response_repairs_trailing_commas_and_unquoted_keys(): + malformed = '{tripName: "Peru Classic", duration: 15,}' + result = _parse_json_response(malformed, 'model-a', 'test') + assert result == {'tripName': 'Peru Classic', 'duration': 15} + + +def test_parse_json_response_raises_when_unparseable(): + with pytest.raises(ValueError): + _parse_json_response('not json at all !!!', 'model-a', 'test') + + +class _FakeChoice: + def __init__(self, content): + self.message = type('Msg', (), {'content': content})() + + +class _FakeCompletionResponse: + def __init__(self, content): + self.choices = [_FakeChoice(content)] + + +def test_call_openrouter_raises_rate_limit_error(monkeypatch): + class _FakeClient: + class chat: + class completions: + @staticmethod + def create(**kwargs): + raise Exception('429 Too Many Requests: rate_limit_exceeded') + + monkeypatch.setattr(extractor, 'OpenAI', lambda api_key, base_url: _FakeClient()) + with pytest.raises(RateLimitError): + extractor._call_openrouter('some-model', 'prompt text', '') + + +def test_call_openrouter_reraises_other_errors(monkeypatch): + class _FakeClient: + class chat: + class completions: + @staticmethod + def create(**kwargs): + raise Exception('500 server error') + + monkeypatch.setattr(extractor, 'OpenAI', lambda api_key, base_url: _FakeClient()) + with pytest.raises(Exception, match='500 server error'): + extractor._call_openrouter('some-model', 'prompt text', '') + + +def test_call_openrouter_appends_content_when_present(monkeypatch): + captured = {} + + class _FakeClient: + class chat: + class completions: + @staticmethod + def create(**kwargs): + captured['messages'] = kwargs['messages'] + return _FakeCompletionResponse('{"ok": true}') + + monkeypatch.setattr(extractor, 'OpenAI', lambda api_key, base_url: _FakeClient()) + result = extractor._call_openrouter('some-model', 'BASE PROMPT', 'EXTRA CONTENT') + + assert result == {'ok': True} + assert 'BASE PROMPT' in captured['messages'][0]['content'] + assert 'EXTRA CONTENT' in captured['messages'][0]['content'] + + +def test_extract_with_openrouter_tries_next_model_on_rate_limit(monkeypatch): + monkeypatch.setattr(extractor, 'FREE_MODELS', ['model-free-1', 'model-free-2']) + monkeypatch.setattr(extractor, 'FALLBACK_MODEL', 'model-paid') + + calls = [] + + def _fake_call(model, prompt, content): + calls.append(model) + if model == 'model-free-1': + raise RateLimitError('rate limited') + return {'tripName': 'ok'} + + monkeypatch.setattr(extractor, '_call_openrouter', _fake_call) + + result = extract_with_openrouter('prompt') + + assert result == {'tripName': 'ok'} + assert calls == ['model-free-1', 'model-free-2'] + + +def test_extract_with_openrouter_falls_back_to_paid_model_and_logs_sentry(monkeypatch): + monkeypatch.setattr(extractor, 'FREE_MODELS', ['model-free-1']) + monkeypatch.setattr(extractor, 'FALLBACK_MODEL', 'model-paid') + + def _fake_call(model, prompt, content): + if model == 'model-free-1': + raise Exception('exhausted') + return {'tripName': 'from-fallback'} + + monkeypatch.setattr(extractor, '_call_openrouter', _fake_call) + sentry_calls = [] + monkeypatch.setattr(extractor.sentry_sdk, 'capture_message', lambda msg, level=None: sentry_calls.append(msg)) + + result = extract_with_openrouter('prompt') + + assert result == {'tripName': 'from-fallback'} + assert sentry_calls == ['OpenRouter paid fallback used'] + + +def test_extract_with_openrouter_raises_when_all_models_exhausted(monkeypatch): + monkeypatch.setattr(extractor, 'FREE_MODELS', ['model-free-1']) + monkeypatch.setattr(extractor, 'FALLBACK_MODEL', 'model-paid') + monkeypatch.setattr(extractor, '_call_openrouter', lambda *a, **k: (_ for _ in ()).throw(Exception('down'))) + + with pytest.raises(ExtractorError): + extract_with_openrouter('prompt') diff --git a/backend/tests/core/test_proxy_service.py b/backend/tests/core/test_proxy_service.py new file mode 100644 index 0000000..f1c2463 --- /dev/null +++ b/backend/tests/core/test_proxy_service.py @@ -0,0 +1,87 @@ +from core.proxy_service import ProxyService + + +def test_webshare_requests_config_with_country(): + service = ProxyService() + config = service._webshare_requests(country='gb') + assert '-GB-rotate' in config['http'] + assert config['http'] == config['https'] + + +def test_webshare_requests_config_without_country(): + service = ProxyService() + config = service._webshare_requests() + assert '-rotate' in config['http'] + assert '-GB-' not in config['http'] + + +def test_webshare_playwright_config(): + service = ProxyService() + config = service._webshare_playwright(country='us') + assert config['server'] == 'http://p.webshare.io:80' + assert config['username'].endswith('-US-rotate') + + +def test_brightdata_requests_and_playwright_configs(): + service = ProxyService() + assert service._brightdata_requests()['http'].startswith('http://') + assert service._brightdata_playwright()['server'] == 'http://brd.superproxy.io:22225' + + +def test_get_proxy_for_domain_uses_webshare_when_no_override(monkeypatch): + service = ProxyService() + monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None) + config = service.get_proxy_for_domain('intrepidtravel.com', country='GB') + assert 'p.webshare.io' in config['http'] + + +def test_get_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch): + service = ProxyService() + monkeypatch.setattr( + 'core.proxy_service.proxy_repository.get_by_domain', + lambda domain: {'id': 'o1', 'provider': 'brightdata'} + ) + config = service.get_proxy_for_domain('flashpack.com') + assert 'brd.superproxy.io' in config['http'] + + +def test_get_playwright_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch): + service = ProxyService() + monkeypatch.setattr( + 'core.proxy_service.proxy_repository.get_by_domain', + lambda domain: {'id': 'o1', 'provider': 'brightdata'} + ) + config = service.get_playwright_proxy_for_domain('flashpack.com') + assert config['server'] == 'http://brd.superproxy.io:22225' + + +def test_escalate_to_brightdata_creates_new_override(monkeypatch): + service = ProxyService() + monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None) + created = {} + monkeypatch.setattr('core.proxy_service.proxy_repository.create', lambda data: created.update(data)) + + service.escalate_to_brightdata('newlyblocked.com') + + assert created['domain'] == 'newlyblocked.com' + assert created['provider'] == 'brightdata' + assert created['blocked_count'] == 1 + + +def test_escalate_to_brightdata_increments_existing_override(monkeypatch): + service = ProxyService() + monkeypatch.setattr( + 'core.proxy_service.proxy_repository.get_by_domain', + lambda domain: {'id': 'o1', 'blocked_count': 2} + ) + updated = {} + monkeypatch.setattr( + 'core.proxy_service.proxy_repository.update', + lambda override_id, data: updated.update({'id': override_id, **data}) + ) + + service.escalate_to_brightdata('repeatoffender.com') + + assert updated['id'] == 'o1' + assert updated['blocked_count'] == 3 + assert updated['provider'] == 'brightdata' diff --git a/backend/tests/core/test_scraper.py b/backend/tests/core/test_scraper.py new file mode 100644 index 0000000..7721e16 --- /dev/null +++ b/backend/tests/core/test_scraper.py @@ -0,0 +1,192 @@ +import asyncio +from core import scraper + + +def _run_async(coro): + """ + Runs a coroutine on a fresh, explicit event loop — mirrors exactly + how core.scraper.scrape() drives render_page() in production. + Deliberately NOT asyncio.run(): scrape()'s own nest_asyncio.apply() + call (exercised by other tests in this file) patches asyncio's + "current event loop" machinery process-wide, which makes a bare + asyncio.run() here liable to pick up and try to reuse a closed loop + left over from an earlier scrape() test, depending on test order. + """ + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +def test_extract_domain_strips_scheme_and_www(): + assert scraper.extract_domain('https://www.intrepidtravel.com/peru') == 'intrepidtravel.com' + assert scraper.extract_domain('intrepidtravel.com') == 'intrepidtravel.com' + + +def test_clean_url_strips_tracking_params(): + dirty = 'https://intrepidtravel.com/peru?utm_source=fb&utm_campaign=x&keep=1' + clean = scraper.clean_url(dirty) + assert 'utm_source' not in clean + assert 'utm_campaign' not in clean + assert 'keep=1' in clean + + +def test_clean_url_strips_fragment(): + assert scraper.clean_url('https://intrepidtravel.com/peru#reviews') == 'https://intrepidtravel.com/peru' + + +def test_clean_url_unchanged_when_already_clean(): + url = 'https://intrepidtravel.com/peru' + assert scraper.clean_url(url) == url + + +def test_is_blocked_true_on_low_char_count(): + assert scraper._is_blocked({'char_count': 100, 'text': ''}) is True + + +def test_is_blocked_true_on_403_status(): + assert scraper._is_blocked({'char_count': 2000, 'status_code': 403, 'text': ''}) is True + + +def test_is_blocked_true_on_captcha_text(): + assert scraper._is_blocked({'char_count': 2000, 'text': 'Please complete this CAPTCHA to continue'}) is True + + +def test_is_blocked_false_for_normal_content(): + assert scraper._is_blocked({'char_count': 5000, 'status_code': 200, 'text': 'Peru Classic 15 days...'}) is False + + +def test_check_robots_txt_permissive_default_on_fetch_failure(monkeypatch): + class _RaisingParser: + def set_url(self, url): + pass + + def read(self): + raise Exception('unreachable') + + monkeypatch.setattr(scraper, 'RobotFileParser', lambda: _RaisingParser()) + assert scraper.check_robots_txt('https://intrepidtravel.com/peru') is True + + +def test_check_robots_txt_respects_disallow(monkeypatch): + class _DisallowParser: + def set_url(self, url): + pass + + def read(self): + pass + + def can_fetch(self, agent, url): + return False + + monkeypatch.setattr(scraper, 'RobotFileParser', lambda: _DisallowParser()) + assert scraper.check_robots_txt('https://intrepidtravel.com/peru') is False + + +def test_scrape_skips_when_robots_txt_disallows(monkeypatch): + monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: False) + result = scraper.scrape('https://intrepidtravel.com/peru') + assert result['success'] is False + assert 'robots.txt' in result['text'] + + +def test_scrape_returns_playwright_result_when_content_is_sufficient(monkeypatch): + monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: True) + + async def _fake_render_page(url, country=None): + return {'text': 'x' * 2000, 'tables': '', 'url': url, 'char_count': 2000, 'success': True} + + monkeypatch.setattr(scraper, 'render_page', _fake_render_page) + + result = scraper.scrape('https://intrepidtravel.com/peru') + assert result['char_count'] == 2000 + assert result['success'] is True + + +def test_scrape_falls_back_to_ai_fetch_when_char_count_low(monkeypatch): + import sys + import types + + monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: True) + + async def _fake_render_page(url, country=None): + return {'text': 'short', 'tables': '', 'url': url, 'char_count': 5, 'success': True} + + monkeypatch.setattr(scraper, 'render_page', _fake_render_page) + + fake_ai_fetch = types.ModuleType('core.ai_fetch') + fake_ai_fetch.fetch_with_ai = lambda url: { + 'text': 'ai-fetched content', 'tables': '', 'url': url, 'char_count': 2000, 'success': True + } + monkeypatch.setitem(sys.modules, 'core.ai_fetch', fake_ai_fetch) + + result = scraper.scrape('https://intrepidtravel.com/peru') + assert result['text'] == 'ai-fetched content' + + +def test_render_page_uses_webshare_proxy_when_not_blocked(monkeypatch): + monkeypatch.setattr( + scraper.proxy_service, 'get_playwright_proxy_for_domain', + lambda domain, country=None: {'server': 'webshare-proxy'} + ) + calls = [] + + async def _fake_render_with_proxy(url, proxy): + calls.append(proxy) + return {'char_count': 5000, 'status_code': 200, 'text': 'good content', 'success': True, 'url': url} + + monkeypatch.setattr(scraper, '_render_with_proxy', _fake_render_with_proxy) + escalated = [] + monkeypatch.setattr(scraper.proxy_service, 'escalate_to_brightdata', lambda domain: escalated.append(domain)) + + result = _run_async(scraper.render_page('https://intrepidtravel.com/peru', country='GB')) + + assert result['success'] is True + assert calls == [{'server': 'webshare-proxy'}] + assert escalated == [] + + +def test_render_page_escalates_to_brightdata_when_blocked(monkeypatch): + monkeypatch.setattr( + scraper.proxy_service, 'get_playwright_proxy_for_domain', + lambda domain, country=None: {'server': 'webshare-proxy'} + ) + monkeypatch.setattr(scraper.proxy_service, '_brightdata_playwright', lambda: {'server': 'brightdata-proxy'}) + + call_log = [] + + async def _fake_render_with_proxy(url, proxy): + call_log.append(proxy) + if proxy['server'] == 'webshare-proxy': + return {'char_count': 50, 'status_code': 403, 'text': 'blocked', 'success': True, 'url': url} + return {'char_count': 5000, 'status_code': 200, 'text': 'good content', 'success': True, 'url': url} + + monkeypatch.setattr(scraper, '_render_with_proxy', _fake_render_with_proxy) + escalated = [] + monkeypatch.setattr(scraper.proxy_service, 'escalate_to_brightdata', lambda domain: escalated.append(domain)) + + result = _run_async(scraper.render_page('https://intrepidtravel.com/peru', country='GB')) + + assert result['text'] == 'good content' + assert escalated == ['intrepidtravel.com'] + assert call_log == [{'server': 'webshare-proxy'}, {'server': 'brightdata-proxy'}] + + +def test_scrape_keeps_playwright_result_when_ai_fetch_returns_none(monkeypatch): + import sys + import types + + monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: True) + + async def _fake_render_page(url, country=None): + return {'text': 'short', 'tables': '', 'url': url, 'char_count': 5, 'success': True} + + monkeypatch.setattr(scraper, 'render_page', _fake_render_page) + + fake_ai_fetch = types.ModuleType('core.ai_fetch') + fake_ai_fetch.fetch_with_ai = lambda url: None + monkeypatch.setitem(sys.modules, 'core.ai_fetch', fake_ai_fetch) + + result = scraper.scrape('https://intrepidtravel.com/peru') + assert result['char_count'] == 5 diff --git a/backend/tests/services/test_analysis_service.py b/backend/tests/services/test_analysis_service.py index 121431b..72593af 100644 --- a/backend/tests/services/test_analysis_service.py +++ b/backend/tests/services/test_analysis_service.py @@ -1,10 +1,10 @@ """ analysis_service orchestrates the fast/refresh path decision (CLAUDE.md -§7). The refresh path lazy-imports Phase 2 modules (core.scraper, -core.extractor, industries.adventure_travel.prompt) that don't exist -yet — _install_fake_module injects stand-ins into sys.modules so the -orchestration logic itself (not the scraping/extraction internals) can -be tested now. +§7). It calls into core.scraper / core.extractor / +industries.adventure_travel.prompt, all of which are now real Phase 2 +modules — _install_fake_module swaps out just the functions this test +needs via monkeypatch.setattr (properly reverted after each test), +rather than mutating the shared module objects directly. """ import sys import types @@ -17,7 +17,15 @@ from repositories.tenant_repository import tenant_repository def _install_fake_module(monkeypatch, dotted_name, **attrs): - """Injects a fake module into sys.modules, creating parent packages as needed.""" + """ + Ensures `dotted_name` resolves to a module (creating fake parent + packages only if truly absent), then uses monkeypatch.setattr for + every attribute — critical when the target is a REAL module (as + core.scraper/core.extractor/industries.adventure_travel.prompt all + are post-Phase-2): monkeypatch.setattr records the original value + and restores it on teardown, whereas a raw setattr would silently + and permanently corrupt the shared module for every later test. + """ parts = dotted_name.split('.') for i in range(1, len(parts) + 1): partial = '.'.join(parts[:i]) @@ -25,7 +33,7 @@ def _install_fake_module(monkeypatch, dotted_name, **attrs): monkeypatch.setitem(sys.modules, partial, types.ModuleType(partial)) target = sys.modules[dotted_name] for key, value in attrs.items(): - setattr(target, key, value) + monkeypatch.setattr(target, key, value) return target diff --git a/backend/tests/services/test_battlecard_service.py b/backend/tests/services/test_battlecard_service.py index 6730a1e..6850a7b 100644 --- a/backend/tests/services/test_battlecard_service.py +++ b/backend/tests/services/test_battlecard_service.py @@ -8,10 +8,13 @@ from repositories.battlecard_repository import battlecard_repository def _install_fake_extractor(monkeypatch, fn): + # core.extractor is now a real Phase 2 module — must use + # monkeypatch.setattr (not a raw setattr) so the swap reverts after + # the test instead of permanently corrupting the shared module. for name in ('core', 'core.extractor'): if name not in sys.modules: monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) - sys.modules['core.extractor'].extract_with_openrouter = fn + monkeypatch.setattr(sys.modules['core.extractor'], 'extract_with_openrouter', fn) def _setup(): diff --git a/backend/tests/test_api_extended.py b/backend/tests/test_api_extended.py index a6c3d88..f3316fa 100644 --- a/backend/tests/test_api_extended.py +++ b/backend/tests/test_api_extended.py @@ -25,13 +25,20 @@ def _make_tenant(**overrides): def _install_fake_module(monkeypatch, dotted_name, **attrs): + """ + Ensures `dotted_name` resolves to a module, then uses + monkeypatch.setattr for every attribute — required now that + core.extractor/industries.adventure_travel.prompt are real Phase 2 + modules: a raw setattr would permanently corrupt the shared module + for every later test since monkeypatch never recorded it. + """ parts = dotted_name.split('.') for i in range(1, len(parts) + 1): partial = '.'.join(parts[:i]) if partial not in sys.modules: monkeypatch.setitem(sys.modules, partial, types.ModuleType(partial)) for key, value in attrs.items(): - setattr(sys.modules[dotted_name], key, value) + monkeypatch.setattr(sys.modules[dotted_name], key, value) # ── Account ──────────────────────────────────────────────────────── diff --git a/backend/tests/test_config_client.py b/backend/tests/test_config_client.py new file mode 100644 index 0000000..1829131 --- /dev/null +++ b/backend/tests/test_config_client.py @@ -0,0 +1,76 @@ +from config import client as config_client + + +class _FakeWorksheet: + def __init__(self, records=None): + self._records = records or [] + self.batch_updates = [] + self.cell_updates = [] + + def batch_update(self, updates): + self.batch_updates.append(updates) + + def get_all_records(self): + return self._records + + def update_cell(self, row, col, value): + self.cell_updates.append((row, col, value)) + + +class _FakeSheet: + def __init__(self, worksheets): + self._worksheets = worksheets + + def worksheet(self, name): + return self._worksheets[name] + + +class _FakeGspreadClient: + def __init__(self, sheet): + self._sheet = sheet + + def open_by_key(self, sheet_id): + return self._sheet + + +def test_write_to_sheet_only_writes_non_empty_fields(monkeypatch): + ws = _FakeWorksheet() + fake_sheet = _FakeSheet({'Standard Competitive Tab': ws}) + monkeypatch.setattr(config_client, 'get_sheet_client', lambda: _FakeGspreadClient(fake_sheet)) + + config_client.write_to_sheet( + 'Standard Competitive Tab', col_index=4, + data={'tripName': 'Peru Classic', 'tripCode': None, 'majorityPrice': ''}, + fields={'tripName': 9, 'tripCode': 10, 'majorityPrice': 25} + ) + + assert len(ws.batch_updates) == 1 + updates = ws.batch_updates[0] + assert len(updates) == 1 # only tripName had a non-empty value + assert updates[0]['values'] == [['Peru Classic']] + + +def test_write_to_sheet_skips_batch_update_when_nothing_to_write(monkeypatch): + ws = _FakeWorksheet() + fake_sheet = _FakeSheet({'Standard Competitive Tab': ws}) + monkeypatch.setattr(config_client, 'get_sheet_client', lambda: _FakeGspreadClient(fake_sheet)) + + config_client.write_to_sheet( + 'Standard Competitive Tab', col_index=4, data={'tripName': None}, fields={'tripName': 9} + ) + + assert ws.batch_updates == [] + + +def test_get_competitors_from_db_filters_active_only(monkeypatch): + ws = _FakeWorksheet(records=[ + {'Travel Style': 'Classic', 'Competitor Name': 'Intrepid', 'Website URL': 'https://intrepidtravel.com', 'Active': 'Yes'}, + {'Travel Style': 'Premium', 'Competitor Name': 'Flash Pack', 'Website URL': 'https://flashpack.com', 'Active': 'No'}, + ]) + fake_sheet = _FakeSheet({'Competitor DB': ws}) + monkeypatch.setattr(config_client, 'get_sheet_client', lambda: _FakeGspreadClient(fake_sheet)) + + result = config_client.get_competitors_from_db() + + assert len(result) == 1 + assert result[0]['name'] == 'Intrepid' diff --git a/backend/tests/test_industries_adventure_travel.py b/backend/tests/test_industries_adventure_travel.py new file mode 100644 index 0000000..11a6579 --- /dev/null +++ b/backend/tests/test_industries_adventure_travel.py @@ -0,0 +1,64 @@ +from industries.adventure_travel import fields, prompt, schema + + +def test_field_maps_have_matching_keys_for_shared_fields(): + shared_keys = set(fields.STANDARD_FIELDS) & set(fields.NGS_FIELDS) + assert 'tripName' in shared_keys + assert 'majorityPrice' in shared_keys + assert 'exclusiveAccess' not in fields.STANDARD_FIELDS + assert 'exclusiveAccess' in fields.NGS_FIELDS + + +def test_build_prompt_embeds_page_content_and_context(): + page_data = {'text': 'Peru Classic 15 days itinerary...', 'tables': 'Day | Activity', 'url': 'https://intrepidtravel.com/peru'} + result = prompt.build_prompt('Intrepid Travel', page_data, 'Inca Trail', 'Peru', 15, 'Classic') + + assert 'Intrepid Travel' in result + assert 'Peru Classic 15 days itinerary' in result + assert 'Day | Activity' in result + assert 'Peru' in result + assert '"exclusiveAccess"' not in result + + +def test_build_prompt_includes_ngs_fields_when_flagged(): + page_data = {'text': 'content', 'tables': '', 'url': 'https://example.com'} + result = prompt.build_prompt('Competitor', page_data, 'Product', 'Peru', 10, 'NGS', is_ngs=True) + assert '"exclusiveAccess"' in result + assert '"groupLeader"' in result + assert '"sustainability"' in result + + +def test_build_prompt_handles_missing_page_data(): + result = prompt.build_prompt('Competitor', None, 'Product', 'Peru', 10, 'Classic') + assert '[PAGE CONTENT WILL BE INSERTED HERE]' in result + assert '[COMPETITOR URL]' in result + + +def test_inject_page_content_replaces_placeholders(): + template = ( + 'Scraped from Competitor tour page at: [COMPETITOR URL]\n' + 'PAGE CONTENT:\n[PAGE CONTENT WILL BE INSERTED HERE]' + ) + page_data = {'text': 'Real scraped content', 'tables': '', 'url': 'https://intrepidtravel.com/peru'} + + result = prompt.inject_page_content(template, 'Competitor', page_data) + + assert 'Real scraped content' in result + assert 'https://intrepidtravel.com/peru' in result + assert '[COMPETITOR URL]' not in result + + +def test_inject_page_content_appends_table_data(): + template = 'PAGE CONTENT:\n[PAGE CONTENT WILL BE INSERTED HERE]\nat [COMPETITOR URL]' + page_data = {'text': 'Base text', 'tables': 'Day | Meal', 'url': 'https://example.com'} + + result = prompt.inject_page_content(template, 'Competitor', page_data) + + assert 'TABLE DATA' in result + assert 'Day | Meal' in result + + +def test_ngs_schema_extends_standard_schema(): + assert set(schema.STANDARD_SCHEMA).issubset(set(schema.NGS_SCHEMA)) + assert 'exclusiveAccess' in schema.NGS_SCHEMA + assert 'exclusiveAccess' not in schema.STANDARD_SCHEMA