Phase 2: refactor legacy app.py into core/industries/config structure
Implements CLAUDE.md Build Order Phase 2 — same scraping/extraction
logic as the legacy app.py, reorganized into the three-layer structure
and upgraded per §10's explicit instructions:
- core/scraper.py — playwright-stealth replaces the manual
add_init_script stealth block; smart proxy (Webshare→Bright Data)
wired via core/proxy_service.py with block-detection + auto-escalation;
check_robots_txt() gate added per §30
- core/extractor.py — OpenRouter model cycling, merging the legacy
JSON-repair fallback chain with the spec's env-driven free/paid
model list and Sentry breadcrumb on paid-fallback use
- core/ai_fetch.py — AI web fetch fallback, ported unchanged
- industries/adventure_travel/{fields,prompt}.py ported unchanged;
schema.py added as the future V2 custom-fields attachment point
- config/client.py — gspread Sheet write-back, ported unchanged
- services/analysis_service.py now sources CACHE_STALENESS_HOURS from
core/scraper.py, its canonical location per §7
Dropped the legacy's commented-out Groq/Gemini alternates — OpenRouter
is the sole active provider and the dead code added nothing.
requirements.txt: relaxed playwright to >=1.55.0 and pinned
playwright-stealth==1.0.6 + setuptools<81 — Python 3.14 has no
prebuilt wheels for the originally-listed pins, and playwright-stealth
2.x replaced the stealth_async(page) API §10 documents verbatim with a
class-based one.
192 tests, 94% coverage. Also fixes a real bug surfaced by this phase:
several test helpers mutated core.extractor / industries.adventure_travel.prompt
via raw setattr instead of monkeypatch.setattr, silently corrupting
shared module state across tests once those modules became real.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
f7aaafa533
commit
6a259a5012
|
|
@ -0,0 +1,62 @@
|
|||
import os
|
||||
import json
|
||||
import gspread
|
||||
from google.oauth2.service_account import Credentials
|
||||
|
||||
# Client-specific Sheet configuration — swapped per client within the
|
||||
# adventure_travel vertical. Ported unchanged from the legacy app.py.
|
||||
SHEET_ID = os.getenv('SHEET_ID')
|
||||
STANDARD_TAB = os.getenv('STANDARD_TAB', 'Standard Competitive Tab')
|
||||
NGS_TAB = os.getenv('NGS_TAB', 'NGS Competitive Tab')
|
||||
SERVICE_ACCOUNT_JSON = os.getenv('GOOGLE_SERVICE_ACCOUNT_JSON')
|
||||
|
||||
|
||||
def get_sheet_client():
|
||||
"""Authorizes a gspread client against the Google service account credentials."""
|
||||
scopes = [
|
||||
'https://www.googleapis.com/auth/spreadsheets',
|
||||
'https://www.googleapis.com/auth/drive',
|
||||
]
|
||||
info = json.loads(SERVICE_ACCOUNT_JSON)
|
||||
creds = Credentials.from_service_account_info(info, scopes=scopes)
|
||||
return gspread.authorize(creds)
|
||||
|
||||
|
||||
def write_to_sheet(tab_name, col_index, data, fields):
|
||||
"""
|
||||
Writes one competitor's extracted fields into a single column of
|
||||
the client Sheet, using the field→row map for this tab type
|
||||
(STANDARD_FIELDS or NGS_FIELDS).
|
||||
|
||||
Data flow:
|
||||
tab_name + col_index + extracted data dict + field row map →
|
||||
batch_update() with one cell write per non-empty field →
|
||||
Sheet updated in a single API call
|
||||
"""
|
||||
client = get_sheet_client()
|
||||
sheet = client.open_by_key(SHEET_ID)
|
||||
ws = sheet.worksheet(tab_name)
|
||||
updates = []
|
||||
for field, row in fields.items():
|
||||
value = data.get(field)
|
||||
if value is None or value == '':
|
||||
continue
|
||||
updates.append({
|
||||
'range': gspread.utils.rowcol_to_a1(row, col_index),
|
||||
'values': [[value]],
|
||||
})
|
||||
if updates:
|
||||
ws.batch_update(updates)
|
||||
|
||||
|
||||
def get_competitors_from_db():
|
||||
"""Reads the client's 'Competitor DB' tab and returns only the active rows."""
|
||||
client = get_sheet_client()
|
||||
sheet = client.open_by_key(SHEET_ID)
|
||||
ws = sheet.worksheet('Competitor DB')
|
||||
rows = ws.get_all_records()
|
||||
return [
|
||||
{'style': r['Travel Style'], 'name': r['Competitor Name'], 'url': r['Website URL'], 'active': r['Active']}
|
||||
for r in rows
|
||||
if str(r.get('Active', '')).strip().lower() == 'yes'
|
||||
]
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
import os
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
from openai import OpenAI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
|
||||
|
||||
# Note: this fallback calls OpenRouter's server-side web_search tool
|
||||
# (Exa engine) rather than making a direct HTTP request to the
|
||||
# competitor's domain — the search itself runs on OpenRouter's
|
||||
# infrastructure, not ours, so core/proxy_service.py's Webshare/Bright
|
||||
# Data routing has no attachment point here. Ported unchanged from the
|
||||
# legacy app.py per CLAUDE.md §10 ("logic stays identical").
|
||||
|
||||
|
||||
def fetch_with_ai(url):
|
||||
"""
|
||||
AI web fetch fallback for bot-protected sites. Fires only when
|
||||
Playwright returns under 1000 chars. Uses OpenRouter's web_search
|
||||
server tool with the Exa engine, restricted to the competitor's
|
||||
own domain. Cost: ~$0.02/call + tokens — only fires for
|
||||
bot-protected sites, not every scrape.
|
||||
|
||||
Data flow:
|
||||
url → domain extracted → OpenRouter chat completion with
|
||||
web_search tool (allowed_domains=[domain]) →
|
||||
response text + any citation content →
|
||||
{ text, tables: '', url, char_count, success } or None if the
|
||||
fallback also returns too little content
|
||||
"""
|
||||
logger.info(f'Playwright failed — attempting AI web fetch for {url}')
|
||||
try:
|
||||
client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1')
|
||||
domain = urlparse(url).netloc
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model='openai/gpt-4o-mini',
|
||||
messages=[{
|
||||
'role': 'user',
|
||||
'content': (
|
||||
'I am a travel industry analyst conducting competitive research. '
|
||||
'Please search for this tour page and extract the following factual data points: '
|
||||
'trip name, trip code, price, duration in days, maximum group size, '
|
||||
'total included meals, hotel names, departure dates, start location, '
|
||||
'end location, service level, and key activities. '
|
||||
f'Return all factual information you find.\n\nURL: {url}'
|
||||
)
|
||||
}],
|
||||
tools=[{
|
||||
'type': 'openrouter:web_search',
|
||||
'parameters': {
|
||||
'engine': 'exa',
|
||||
'max_results': 5,
|
||||
'search_context_size': 'high',
|
||||
'allowed_domains': [domain],
|
||||
}
|
||||
}],
|
||||
temperature=0.0,
|
||||
max_tokens=8192
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
text = msg.content if isinstance(msg.content, str) else ''
|
||||
|
||||
# Also pull content from tool-result annotations if present
|
||||
if hasattr(msg, 'annotations') and msg.annotations:
|
||||
for annotation in msg.annotations:
|
||||
if hasattr(annotation, 'url_citation') and annotation.url_citation:
|
||||
citation_content = getattr(annotation.url_citation, 'content', '')
|
||||
if citation_content:
|
||||
text = text + '\n\n' + citation_content
|
||||
|
||||
text = text.strip()
|
||||
char_count = len(text)
|
||||
logger.info(f'AI web fetch returned {char_count} chars for {url}')
|
||||
|
||||
if char_count > 300:
|
||||
return {'text': text[:25000], 'tables': '', 'url': url, 'char_count': char_count, 'success': True}
|
||||
|
||||
logger.warning(f'AI web fetch also returned low content for {url}')
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'AI web fetch failed for {url}: {str(e)}')
|
||||
return None
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
import os
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
from openai import OpenAI
|
||||
import sentry_sdk
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
|
||||
FREE_MODELS = [m.strip() for m in os.getenv('OPENROUTER_FREE_MODELS', '').split(',') if m.strip()]
|
||||
FALLBACK_MODEL = os.getenv('OPENROUTER_FALLBACK_MODEL', 'anthropic/claude-haiku-4-5')
|
||||
|
||||
|
||||
class ExtractorError(Exception):
|
||||
"""Raised when every OpenRouter model (free tier + paid fallback) has failed."""
|
||||
pass
|
||||
|
||||
|
||||
class RateLimitError(Exception):
|
||||
"""Raised internally when an OpenRouter model returns a 429 / rate-limit response."""
|
||||
pass
|
||||
|
||||
|
||||
def _parse_json_response(text, model, label):
|
||||
"""
|
||||
Parses a model's raw text response into JSON via progressively more
|
||||
aggressive repairs — ported unchanged from the legacy app.py
|
||||
extraction logic: raw parse → regex object extraction →
|
||||
trailing-comma/unquoted-key repair.
|
||||
|
||||
Data flow:
|
||||
raw model text → strip code fences →
|
||||
json.loads() (fast path) →
|
||||
regex-extract the outermost {...} block and retry →
|
||||
strip trailing commas + quote bare keys and retry →
|
||||
parsed dict returned, or ValueError if all three fail
|
||||
"""
|
||||
cleaned = re.sub(r'```json|```', '', text).strip()
|
||||
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
match = re.search(r'\{[\s\S]*\}', cleaned)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
fixed = re.sub(r',\s*}', '}', cleaned)
|
||||
fixed = re.sub(r',\s*]', ']', fixed)
|
||||
fixed = re.sub(r'([{,])\s*([a-zA-Z0-9_]+)\s*:', r'\1"\2":', fixed)
|
||||
return json.loads(fixed)
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError(f'Could not parse response from {model} for {label}')
|
||||
|
||||
|
||||
def _call_openrouter(model, prompt, content):
|
||||
"""
|
||||
Issues one chat completion request to OpenRouter for `model`.
|
||||
`content` is appended to `prompt` when present — most callers
|
||||
(via industries.adventure_travel.prompt.build_prompt) already embed
|
||||
page content directly into `prompt`, so `content` is typically
|
||||
empty; analysis_service's refresh path passes it explicitly too,
|
||||
which is harmless (the model just sees the page text once via the
|
||||
prompt and, redundantly, once more here).
|
||||
|
||||
Data flow:
|
||||
model + prompt + content → OpenAI client (OpenRouter base_url) →
|
||||
chat.completions.create() → raise RateLimitError on 429/rate-limit
|
||||
responses (caller moves to the next model) → raw text →
|
||||
_parse_json_response() → parsed dict returned
|
||||
"""
|
||||
client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1')
|
||||
message = f'{prompt}\n\n{content}' if content else prompt
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{'role': 'user', 'content': message}],
|
||||
temperature=0.1,
|
||||
max_tokens=4096
|
||||
)
|
||||
except Exception as e:
|
||||
error_str = str(e)
|
||||
if '429' in error_str or 'rate_limit' in error_str.lower():
|
||||
raise RateLimitError(error_str)
|
||||
raise
|
||||
|
||||
text = response.choices[0].message.content
|
||||
return _parse_json_response(text, model, 'extraction')
|
||||
|
||||
|
||||
def extract_with_openrouter(prompt, content=''):
|
||||
"""
|
||||
Cycles through free OpenRouter models in sequence. Falls back to
|
||||
the paid fallback model if all free models fail or rate-limit.
|
||||
Never returns None — raises ExtractorError only after every model
|
||||
(free + paid) has failed.
|
||||
|
||||
Data flow:
|
||||
prompt + content → for each model in FREE_MODELS + [FALLBACK_MODEL]:
|
||||
_call_openrouter() → success: return parsed JSON
|
||||
(logging + a Sentry breadcrumb if the paid fallback had to be used) →
|
||||
RateLimitError: log, try next model →
|
||||
any other error: log, try next model →
|
||||
all models exhausted → raise ExtractorError
|
||||
"""
|
||||
models_to_try = FREE_MODELS + [FALLBACK_MODEL]
|
||||
|
||||
for model in models_to_try:
|
||||
try:
|
||||
result = _call_openrouter(model, prompt, content)
|
||||
if model == FALLBACK_MODEL and FREE_MODELS:
|
||||
logger.warning(f'Free models exhausted — used paid fallback: {model}')
|
||||
sentry_sdk.capture_message('OpenRouter paid fallback used', level='warning')
|
||||
return result
|
||||
except RateLimitError:
|
||||
logger.warning(f'Rate limit on {model} — trying next')
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f'Model {model} failed: {str(e)}')
|
||||
continue
|
||||
|
||||
raise ExtractorError('All OpenRouter models exhausted including paid fallback')
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
import os
|
||||
from datetime import datetime, timezone
|
||||
from repositories.proxy_repository import proxy_repository
|
||||
|
||||
WEBSHARE_USER = os.getenv('WEBSHARE_USER')
|
||||
WEBSHARE_PASS = os.getenv('WEBSHARE_PASS')
|
||||
BRIGHTDATA_USER = os.getenv('BRIGHTDATA_USER')
|
||||
BRIGHTDATA_PASS = os.getenv('BRIGHTDATA_PASS')
|
||||
|
||||
|
||||
class ProxyService:
|
||||
"""
|
||||
Smart proxy provider — uses Webshare by default with country
|
||||
targeting, auto-escalates to Bright Data for domains that block
|
||||
Webshare. This is the ONLY place in the codebase that knows about
|
||||
any proxy provider — swapping providers or switching logic means
|
||||
changing only this file (CLAUDE.md §10).
|
||||
|
||||
Country targeting uses competitors.primary_market (ISO code e.g.
|
||||
GB, US, AU) passed through from the competitor record — no
|
||||
hardcoded country map in code.
|
||||
|
||||
Data flow (first request to a domain):
|
||||
domain + country → proxy_overrides lookup → no override found →
|
||||
Webshare config with country targeting returned →
|
||||
scraper attempts request → if blocked →
|
||||
escalate_to_brightdata(domain) called →
|
||||
Bright Data config returned → scraper retries →
|
||||
override stored in PocketBase for this domain
|
||||
|
||||
Data flow (subsequent requests to a known blocked domain):
|
||||
domain + country → proxy_overrides lookup → brightdata override
|
||||
found → Bright Data config returned immediately, no Webshare attempt
|
||||
"""
|
||||
|
||||
def get_proxy_for_domain(self, domain, country=None):
|
||||
"""Returns a `requests`-style proxy config, routed to Bright Data if previously blocked."""
|
||||
override = proxy_repository.get_by_domain(domain)
|
||||
if override and override.get('provider') == 'brightdata':
|
||||
return self._brightdata_requests()
|
||||
return self._webshare_requests(country=country)
|
||||
|
||||
def get_playwright_proxy_for_domain(self, domain, country=None):
|
||||
"""Returns a Playwright proxy config, routed to Bright Data if previously blocked."""
|
||||
override = proxy_repository.get_by_domain(domain)
|
||||
if override and override.get('provider') == 'brightdata':
|
||||
return self._brightdata_playwright()
|
||||
return self._webshare_playwright(country=country)
|
||||
|
||||
def escalate_to_brightdata(self, domain):
|
||||
"""
|
||||
Records that Webshare was blocked on this domain. Upserts a
|
||||
proxy_override so future requests skip Webshare entirely for
|
||||
this domain.
|
||||
"""
|
||||
existing = proxy_repository.get_by_domain(domain)
|
||||
if existing:
|
||||
proxy_repository.update(existing['id'], {
|
||||
'provider': 'brightdata',
|
||||
'blocked_count': existing.get('blocked_count', 0) + 1,
|
||||
'last_blocked': datetime.now(timezone.utc).isoformat(),
|
||||
})
|
||||
else:
|
||||
proxy_repository.create({
|
||||
'domain': domain,
|
||||
'provider': 'brightdata',
|
||||
'blocked_count': 1,
|
||||
'last_blocked': datetime.now(timezone.utc).isoformat(),
|
||||
})
|
||||
|
||||
def _webshare_requests(self, country=None):
|
||||
"""Webshare `requests` proxy config. Format: username-{COUNTRY}-rotate. No country = full pool rotation."""
|
||||
suffix = f'-{country.upper()}-rotate' if country else '-rotate'
|
||||
username = f'{WEBSHARE_USER}{suffix}'
|
||||
return {
|
||||
'http': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80',
|
||||
'https': f'http://{username}:{WEBSHARE_PASS}@p.webshare.io:80',
|
||||
}
|
||||
|
||||
def _webshare_playwright(self, country=None):
|
||||
"""Webshare Playwright proxy config with optional country targeting."""
|
||||
suffix = f'-{country.upper()}-rotate' if country else '-rotate'
|
||||
return {
|
||||
'server': 'http://p.webshare.io:80',
|
||||
'username': f'{WEBSHARE_USER}{suffix}',
|
||||
'password': WEBSHARE_PASS,
|
||||
}
|
||||
|
||||
def _brightdata_requests(self):
|
||||
return {
|
||||
'http': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:22225',
|
||||
'https': f'http://{BRIGHTDATA_USER}:{BRIGHTDATA_PASS}@brd.superproxy.io:22225',
|
||||
}
|
||||
|
||||
def _brightdata_playwright(self):
|
||||
return {
|
||||
'server': 'http://brd.superproxy.io:22225',
|
||||
'username': BRIGHTDATA_USER,
|
||||
'password': BRIGHTDATA_PASS,
|
||||
}
|
||||
|
||||
|
||||
proxy_service = ProxyService()
|
||||
|
|
@ -0,0 +1,293 @@
|
|||
import re
|
||||
import asyncio
|
||||
import logging
|
||||
from urllib.parse import urlparse, urlencode, parse_qs, urlunparse
|
||||
from urllib.robotparser import RobotFileParser
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from playwright_stealth import stealth_async
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core.proxy_service import proxy_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default staleness threshold for the on-demand fast/refresh path
|
||||
# decision (CLAUDE.md §7). This is the canonical, adjustable location
|
||||
# per §7 — services/analysis_service.py imports it from here rather
|
||||
# than redefining it.
|
||||
CACHE_STALENESS_HOURS = 24
|
||||
|
||||
# Block only known third-party analytics/ad domains — never block the
|
||||
# competitor's own first-party assets.
|
||||
BLOCKED_DOMAINS = [
|
||||
'google-analytics.com', 'googletagmanager.com', 'doubleclick.net',
|
||||
'facebook.net', 'facebook.com/tr', 'hotjar.com', 'intercom.io',
|
||||
'zendesk.com', 'hubspot.com', 'marketo.com', 'segment.com',
|
||||
'amplitude.com', 'mixpanel.com', 'optimizely.com', 'crazyegg.com',
|
||||
]
|
||||
|
||||
|
||||
def extract_domain(url):
|
||||
"""Strips scheme/path down to a bare domain — used for proxy_overrides and robots.txt lookups."""
|
||||
return urlparse(url if '://' in url else f'//{url}').netloc.lower().lstrip('www.')
|
||||
|
||||
|
||||
def check_robots_txt(url):
|
||||
"""
|
||||
Checks if the given URL is allowed to be scraped per robots.txt.
|
||||
Returns True if scraping is permitted, False if disallowed.
|
||||
Defaults to True if robots.txt cannot be fetched (permissive
|
||||
default) — CLAUDE.md §30 Web Scraping Legal Policy.
|
||||
|
||||
Data flow:
|
||||
url → extract root domain → fetch /robots.txt →
|
||||
parse with urllib.robotparser →
|
||||
check if Bettersight's user-agent is allowed for this path
|
||||
"""
|
||||
root = f'{urlparse(url).scheme}://{urlparse(url).netloc}'
|
||||
parser = RobotFileParser()
|
||||
parser.set_url(f'{root}/robots.txt')
|
||||
try:
|
||||
parser.read()
|
||||
return parser.can_fetch('*', url)
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
def clean_url(url):
|
||||
"""Strips URL fragments and common tracking parameters."""
|
||||
parsed = urlparse(url)
|
||||
strip_params = {
|
||||
'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
|
||||
'referrerView', 'ref', 'referrer', 'source', 'fbclid', 'gclid',
|
||||
'mc_cid', 'mc_eid', '_ga', 'affiliate',
|
||||
}
|
||||
if parsed.query:
|
||||
params = parse_qs(parsed.query, keep_blank_values=True)
|
||||
cleaned = {k: v for k, v in params.items() if k not in strip_params}
|
||||
query = urlencode(cleaned, doseq=True)
|
||||
else:
|
||||
query = ''
|
||||
clean = urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, query, ''))
|
||||
if clean != url:
|
||||
logger.info(f'URL cleaned: {url} → {clean}')
|
||||
return clean
|
||||
|
||||
|
||||
def _is_blocked(result):
|
||||
"""
|
||||
Detects common block signals from a scrape result. Returns True if
|
||||
the response looks like a bot wall rather than real content.
|
||||
Signals: low char count, HTTP 403/429, Cloudflare challenge, CAPTCHA text.
|
||||
"""
|
||||
if result.get('char_count', 0) < 500:
|
||||
return True
|
||||
if result.get('status_code') in (403, 429):
|
||||
return True
|
||||
block_signals = ['cf-browser-verification', 'captcha', 'access denied', 'blocked', 'robot']
|
||||
return any(signal in result.get('text', '').lower() for signal in block_signals)
|
||||
|
||||
|
||||
async def _render_with_proxy(url, proxy):
|
||||
"""
|
||||
Launches a stealth Playwright browser through the given proxy
|
||||
config and extracts cleaned page text + tables. This is the actual
|
||||
page-rendering logic — split out from render_page() so it can be
|
||||
retried with a different proxy (Bright Data) on the first attempt
|
||||
being blocked, without duplicating the browser setup.
|
||||
|
||||
Data flow:
|
||||
url + proxy config → Playwright chromium launch (proxied) →
|
||||
playwright-stealth applied → navigate → dismiss cookie banner →
|
||||
wait for content → scroll to trigger lazy-loading →
|
||||
HTML → BeautifulSoup cleaning → { text, tables, url, char_count, success }
|
||||
"""
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True,
|
||||
proxy=proxy,
|
||||
args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
|
||||
)
|
||||
context = await browser.new_context(
|
||||
user_agent=(
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
|
||||
),
|
||||
viewport={'width': 1366, 'height': 768},
|
||||
locale='en-GB',
|
||||
timezone_id='Europe/London',
|
||||
extra_http_headers={
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-GB,en;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'DNT': '1',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
}
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
# Comprehensive stealth (navigator.webdriver, plugins, canvas/WebGL
|
||||
# fingerprint, audio context, chrome runtime, ~20 evasions total)
|
||||
# replaces the old manual add_init_script block per CLAUDE.md §10.
|
||||
await stealth_async(page)
|
||||
|
||||
async def block_trackers(route):
|
||||
if any(domain in route.request.url for domain in BLOCKED_DOMAINS):
|
||||
await route.abort()
|
||||
else:
|
||||
await route.continue_()
|
||||
|
||||
await page.route('**/*', block_trackers)
|
||||
|
||||
try:
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||
logger.info(f'Loaded: {url}')
|
||||
except Exception as e:
|
||||
logger.warning(f'domcontentloaded failed, trying networkidle: {str(e)[:100]}')
|
||||
await page.goto(url, wait_until='networkidle', timeout=60000)
|
||||
|
||||
# Dismiss cookie consent banners
|
||||
try:
|
||||
await page.click(
|
||||
'button:has-text("Accept All"), button:has-text("Accept all"), '
|
||||
'button:has-text("Accept"), button:has-text("OK"), '
|
||||
'button:has-text("I agree"), button:has-text("Agree"), '
|
||||
'[id*="cookie"] button, [class*="cookie"] button, '
|
||||
'[id*="consent"] button, [class*="consent"] button',
|
||||
timeout=3000
|
||||
)
|
||||
logger.info('Cookie banner dismissed')
|
||||
await page.wait_for_timeout(1000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Wait for meaningful content
|
||||
try:
|
||||
await page.wait_for_selector(
|
||||
'h1, .trip-overview, main, article, #overview, '
|
||||
'[class*="itinerary"], [class*="price"], [class*="trip"]',
|
||||
timeout=8000
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
# Two scroll passes to trigger lazy-loaded content
|
||||
for _ in range(2):
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||||
await page.wait_for_timeout(1500)
|
||||
await page.evaluate('window.scrollTo(0, 0)')
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||||
await page.wait_for_timeout(2000)
|
||||
|
||||
html = await page.content()
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
|
||||
for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'iframe', 'noscript', 'svg', 'button', 'form']):
|
||||
tag.decompose()
|
||||
|
||||
main_content = (
|
||||
soup.find('main') or soup.find('article') or soup.find(id='main')
|
||||
or soup.find(id='content') or soup.find(class_='content') or soup
|
||||
)
|
||||
|
||||
text = main_content.get_text(separator=' ', strip=True)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
tables = [t.get_text(separator=' | ', strip=True) for t in soup.find_all('table')]
|
||||
table_text = '\n'.join(tables)
|
||||
char_count = len(text)
|
||||
|
||||
if char_count < 500:
|
||||
logger.warning(f'Low char count ({char_count}) for {url} — will attempt AI web fetch fallback')
|
||||
|
||||
return {
|
||||
'text': text[:25000],
|
||||
'tables': table_text[:5000],
|
||||
'url': url,
|
||||
'char_count': char_count,
|
||||
'success': True,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Render failed for {url}: {str(e)}')
|
||||
return {'text': f'Render failed: {str(e)}', 'tables': '', 'url': url, 'char_count': 0, 'success': False}
|
||||
finally:
|
||||
await browser.close()
|
||||
|
||||
|
||||
async def render_page(url, country=None):
|
||||
"""
|
||||
Renders a page using the smart proxy service with country
|
||||
targeting. `country` comes from competitors.primary_market in
|
||||
PocketBase. Detects blocks and auto-escalates to Bright Data if
|
||||
Webshare fails.
|
||||
|
||||
Data flow:
|
||||
url + country → domain extracted →
|
||||
proxy_service.get_playwright_proxy_for_domain(domain, country) →
|
||||
_render_with_proxy() → if blocked → escalate_to_brightdata(domain) →
|
||||
retry once with Bright Data (no country targeting at fallback level)
|
||||
"""
|
||||
domain = extract_domain(url)
|
||||
proxy = proxy_service.get_playwright_proxy_for_domain(domain, country)
|
||||
result = await _render_with_proxy(url, proxy)
|
||||
|
||||
if _is_blocked(result):
|
||||
logger.warning(f'Webshare blocked on {domain} — escalating to Bright Data')
|
||||
proxy_service.escalate_to_brightdata(domain)
|
||||
result = await _render_with_proxy(url, proxy_service._brightdata_playwright())
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def scrape(url, country=None):
|
||||
"""
|
||||
Synchronous entry point for scraping one competitor page. Checks
|
||||
robots.txt first (legal/compliance gate per §30), then renders via
|
||||
Playwright, falling back to the AI web fetch path if Playwright's
|
||||
char count is too low (bot-protected sites).
|
||||
|
||||
Data flow:
|
||||
url + country → clean_url() → check_robots_txt() →
|
||||
disallowed: return failure immediately (no scrape attempted) →
|
||||
allowed: render_page() (sync wrapper via nest_asyncio) →
|
||||
char_count < 1000: fetch_with_ai() fallback →
|
||||
final { text, tables, url, char_count, success } returned
|
||||
"""
|
||||
import nest_asyncio
|
||||
nest_asyncio.apply()
|
||||
|
||||
url = clean_url(url)
|
||||
|
||||
if not check_robots_txt(url):
|
||||
logger.warning(f'robots.txt disallows scraping {url} — skipping')
|
||||
return {
|
||||
'text': 'Scraping disallowed by robots.txt for this URL',
|
||||
'tables': '', 'url': url, 'char_count': 0, 'success': False,
|
||||
}
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
result = loop.run_until_complete(render_page(url, country=country))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
if result.get('char_count', 0) < 1000 and result.get('success'):
|
||||
logger.info(f'Playwright char count low — trying AI web fetch for {url}')
|
||||
from core.ai_fetch import fetch_with_ai
|
||||
ai_result = fetch_with_ai(url)
|
||||
if ai_result:
|
||||
return ai_result
|
||||
|
||||
return result
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
# Column position maps for the legacy Google Sheet write-back
|
||||
# (config/client.py). Ported unchanged from the pre-refactor app.py —
|
||||
# these positions are specific to the existing client Sheet template
|
||||
# and must not be renumbered without also updating that template.
|
||||
|
||||
STANDARD_FIELDS = {
|
||||
'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
|
||||
'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
|
||||
'startLocation': 15, 'endLocation': 16, 'departures': 17,
|
||||
'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
|
||||
'hotels': 22, 'activities': 23, 'relevancy': 24,
|
||||
'majorityPrice': 25, 'priceLow': 26, 'priceHigh': 27,
|
||||
'comments': 30, 'dateUsed': 31,
|
||||
'audPrice': 32, 'cadPrice': 33, 'eurPrice': 34, 'gbpPrice': 35,
|
||||
}
|
||||
|
||||
NGS_FIELDS = {
|
||||
'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
|
||||
'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
|
||||
'startLocation': 15, 'endLocation': 16, 'departures': 17,
|
||||
'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
|
||||
'hotels': 22, 'activities': 23, 'exclusiveAccess': 24,
|
||||
'groupLeader': 25, 'sustainability': 26, 'relevancy': 27,
|
||||
'majorityPrice': 28, 'priceLow': 29, 'priceHigh': 30,
|
||||
'comments': 33, 'dateUsed': 34,
|
||||
'audPrice': 35, 'cadPrice': 36, 'eurPrice': 37, 'gbpPrice': 38,
|
||||
}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
# Prompt template and injection logic for the adventure travel
|
||||
# vertical — ported unchanged from the legacy app.py. Per CLAUDE.md
|
||||
# §10, pivoting to a new industry means adding a new folder under
|
||||
# industries/ with these same three files; core/ is never touched.
|
||||
|
||||
|
||||
def build_prompt(competitor_name, page_data, product_name,
|
||||
destination, duration, travel_style, is_ngs=False):
|
||||
"""
|
||||
Builds the full extraction prompt for one competitor page, embedding
|
||||
the scraped page content directly into the returned template.
|
||||
|
||||
Data flow:
|
||||
competitor_name + page_data (scraped text/tables/url) + client
|
||||
product context (name/destination/duration/style) + is_ngs flag →
|
||||
NGS-specific field block appended if is_ngs →
|
||||
full prompt string returned, ready to send to core.extractor
|
||||
"""
|
||||
ngs_fields = '''
|
||||
"exclusiveAccess": "exclusive access or special unique inclusions",
|
||||
"groupLeader": "group leader and/or local expert description",
|
||||
"sustainability": "sustainability inclusions or null",''' if is_ngs else ''
|
||||
|
||||
content = page_data['text'] if page_data else '[PAGE CONTENT WILL BE INSERTED HERE]'
|
||||
if page_data and page_data.get('tables'):
|
||||
content += f'\n\nTABLE DATA:\n{page_data["tables"]}'
|
||||
|
||||
url = page_data['url'] if page_data else '[COMPETITOR URL]'
|
||||
|
||||
return f'''You are a travel industry analyst helping G Adventures research competitors.
|
||||
Be concise — use short phrases not full sentences for all fields except comments.
|
||||
Read the ENTIRE page content carefully before extracting any fields.
|
||||
Pay particular attention to itinerary sections, departure tables, and pricing grids
|
||||
which may appear later in the content.
|
||||
|
||||
The content below was scraped from the {competitor_name} tour page at: {url}
|
||||
|
||||
This is the specific page selected as most comparable to:
|
||||
- G Adventures Product: {product_name}
|
||||
- Destination: {destination}
|
||||
- Duration: ~{duration} days
|
||||
- G Adventures Travel Style: {travel_style}
|
||||
|
||||
PAGE CONTENT:
|
||||
---
|
||||
{content}
|
||||
---
|
||||
|
||||
CRITICAL PRICING RULES:
|
||||
- majorityPrice: Standard/regular adult price for the most common departure.
|
||||
IGNORE sale, promotional, early bird, or "was/now" pricing. Use non-discounted rack rate only.
|
||||
- Prices may appear as "from $X", "per person from $X", or inside a departures table or pricing grid.
|
||||
Extract the most commonly listed standard adult price regardless of how it is labelled on the page.
|
||||
- priceLow: Lowest standard price across all departures shown (not sale price).
|
||||
- priceHigh: Highest standard price across all departures shown (not sale price).
|
||||
- If a site requires clicking "see more departures" to show all dates, note this in seasonality
|
||||
and use only prices visible on the main listing page.
|
||||
- Multi-currency: only fill if the price is explicitly listed in that currency on the page.
|
||||
Do not convert. If the page shows GBP only, fill gbpPrice and leave USD fields null.
|
||||
|
||||
MEALS RULE:
|
||||
- meals: Return the TOTAL number of included meals as a single integer only. No text description.
|
||||
- Meals may be listed individually per day in the itinerary section
|
||||
(e.g. "Day 1: Breakfast", "Day 3: Breakfast and Dinner").
|
||||
Count every individual meal mention across all itinerary days.
|
||||
Do not rely on a summary box — read through the full itinerary to count.
|
||||
- Count each meal type separately: 1 breakfast + 1 lunch + 1 dinner = 3 meals.
|
||||
|
||||
Return ONLY a valid JSON object. No markdown, no code fences, no explanation.
|
||||
|
||||
{{
|
||||
"company": "{competitor_name}",
|
||||
"link": "{url}",
|
||||
"tripName": "exact trip name from the page",
|
||||
"tripCode": "trip code if shown, otherwise null",
|
||||
"serviceLevel": "accommodation service level",
|
||||
"groupSize": max group size as integer,
|
||||
"duration": number of days as integer,
|
||||
"meals": total included meals as integer,{ngs_fields}
|
||||
"startLocation": "departure city",
|
||||
"endLocation": "end city",
|
||||
"departures": approximate departures per year as integer or null,
|
||||
"seasonality": "price bands by season. Note if more departures are hidden behind a button.",
|
||||
"startDays": "departure days of week",
|
||||
"targetAudience": "target traveller",
|
||||
"hotels": "hotel types or properties",
|
||||
"activities": "key included activities",
|
||||
"relevancy": relevancy score 1-5 as integer,
|
||||
"majorityPrice": standard adult USD price as number or null,
|
||||
"priceLow": lowest standard USD price as number or null,
|
||||
"priceHigh": highest standard USD price as number or null,
|
||||
"comments": "analysis: key differences from G Adventures, strengths, weaknesses, pricing position",
|
||||
"dateUsed": "specific departure date used for majority price",
|
||||
"audPrice": standard AUD price as number or null,
|
||||
"cadPrice": standard CAD price as number or null,
|
||||
"eurPrice": standard EUR price as number or null,
|
||||
"gbpPrice": standard GBP price as number or null
|
||||
}}'''
|
||||
|
||||
|
||||
def inject_page_content(prompt_template, competitor_name, page_data):
|
||||
"""
|
||||
Injects real scraped content into a user-edited prompt template
|
||||
(used when the PM has customised the prompt via the dashboard's
|
||||
prompt preview/edit flow instead of using build_prompt()'s default).
|
||||
"""
|
||||
content = page_data['text']
|
||||
if page_data.get('tables'):
|
||||
content += f'\n\nTABLE DATA:\n{page_data["tables"]}'
|
||||
result = prompt_template.replace('[PAGE CONTENT WILL BE INSERTED HERE]', content)
|
||||
result = result.replace('[COMPETITOR URL]', page_data['url'])
|
||||
result = result.replace(
|
||||
competitor_name + ' tour page at: [COMPETITOR URL]',
|
||||
competitor_name + f' tour page at: {page_data["url"]}'
|
||||
)
|
||||
return result
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# JSON extraction schema for the adventure travel vertical, expressed
|
||||
# as data rather than embedded in the prompt template string. This
|
||||
# mirrors the JSON shape at the end of prompt.build_prompt()'s output
|
||||
# and exists as the attachment point for CLAUDE.md §28's Version 2
|
||||
# custom extraction fields (build_json_schema() appends tenant custom
|
||||
# fields to whichever of these two dicts applies) — that build_json_schema()
|
||||
# function itself is out of scope until Discover/Intelligence tier work
|
||||
# begins; this file only establishes the base schema it would extend.
|
||||
|
||||
STANDARD_SCHEMA = {
|
||||
'company': 'text — competitor name',
|
||||
'link': 'text — page URL',
|
||||
'tripName': 'text — exact trip name from the page',
|
||||
'tripCode': 'text — trip code if shown, otherwise null',
|
||||
'serviceLevel': 'text — accommodation service level',
|
||||
'groupSize': 'integer — max group size',
|
||||
'duration': 'integer — number of days',
|
||||
'meals': 'integer — total included meals',
|
||||
'startLocation': 'text — departure city',
|
||||
'endLocation': 'text — end city',
|
||||
'departures': 'integer — approximate departures per year, or null',
|
||||
'seasonality': 'text — price bands by season',
|
||||
'startDays': 'text — departure days of week',
|
||||
'targetAudience': 'text — target traveller',
|
||||
'hotels': 'text — hotel types or properties',
|
||||
'activities': 'text — key included activities',
|
||||
'relevancy': 'integer — relevancy score 1-5',
|
||||
'majorityPrice': 'price — standard adult USD price or null',
|
||||
'priceLow': 'price — lowest standard USD price or null',
|
||||
'priceHigh': 'price — highest standard USD price or null',
|
||||
'comments': 'text — analysis: differences, strengths, weaknesses, pricing position',
|
||||
'dateUsed': 'text — specific departure date used for majority price',
|
||||
'audPrice': 'price — standard AUD price or null',
|
||||
'cadPrice': 'price — standard CAD price or null',
|
||||
'eurPrice': 'price — standard EUR price or null',
|
||||
'gbpPrice': 'price — standard GBP price or null',
|
||||
}
|
||||
|
||||
NGS_SCHEMA = {
|
||||
**STANDARD_SCHEMA,
|
||||
'exclusiveAccess': 'text — exclusive access or special unique inclusions',
|
||||
'groupLeader': 'text — group leader and/or local expert description',
|
||||
'sustainability': 'text — sustainability inclusions or null',
|
||||
}
|
||||
|
|
@ -14,8 +14,19 @@ stripe==10.7.0
|
|||
resend==2.4.0
|
||||
|
||||
# ── Phase 2 — scraping engine (moved from legacy app.py) ─────
|
||||
playwright==1.46.0
|
||||
# playwright is unpinned to a minimum, not exact — 1.46.0's pinned
|
||||
# greenlet dependency has no prebuilt wheel on newer Python versions
|
||||
# and fails to build from source; newer playwright releases pull in a
|
||||
# newer greenlet that does ship prebuilt wheels.
|
||||
playwright>=1.55.0
|
||||
# playwright-stealth is pinned to 1.0.6 (not the 2.x line) because
|
||||
# CLAUDE.md §10 documents the 1.x `stealth_async(page)` function API
|
||||
# verbatim; playwright-stealth 2.x replaced it with a class-based
|
||||
# `Stealth().apply_stealth_async(...)` API — a breaking change from
|
||||
# what's specified. 1.0.6 depends on pkg_resources, which setuptools
|
||||
# >=81 no longer ships — hence the setuptools pin below.
|
||||
playwright-stealth==1.0.6
|
||||
setuptools<81
|
||||
beautifulsoup4==4.12.3
|
||||
lxml==5.3.0
|
||||
nest_asyncio==1.6.0
|
||||
|
|
|
|||
|
|
@ -5,9 +5,6 @@ from repositories.product_repository import product_repository
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default staleness threshold — configurable here only, per CLAUDE.md §7.
|
||||
CACHE_STALENESS_HOURS = 24
|
||||
|
||||
|
||||
def needs_refresh(competitor):
|
||||
"""
|
||||
|
|
@ -32,6 +29,12 @@ def needs_refresh(competitor):
|
|||
except (ValueError, AttributeError):
|
||||
return True
|
||||
|
||||
# CACHE_STALENESS_HOURS' canonical, adjustable home is core/scraper.py
|
||||
# per CLAUDE.md §7 ("in core/scraper.py — adjust here only") — imported
|
||||
# lazily here for the same reason every other core.* reference in this
|
||||
# file is lazy: this module must stay importable independent of Phase 2.
|
||||
from core.scraper import CACHE_STALENESS_HOURS
|
||||
|
||||
age = datetime.now(timezone.utc) - scraped_at
|
||||
return age > timedelta(hours=CACHE_STALENESS_HOURS)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
from core import ai_fetch
|
||||
|
||||
|
||||
class _FakeMessage:
|
||||
def __init__(self, content, annotations=None):
|
||||
self.content = content
|
||||
self.annotations = annotations
|
||||
|
||||
|
||||
class _FakeChoice:
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, message):
|
||||
self.choices = [_FakeChoice(message)]
|
||||
|
||||
|
||||
def test_fetch_with_ai_returns_result_when_content_sufficient(monkeypatch):
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
return _FakeResponse(_FakeMessage('x' * 400))
|
||||
|
||||
monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient())
|
||||
|
||||
result = ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru')
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['char_count'] == 400
|
||||
|
||||
|
||||
def test_fetch_with_ai_returns_none_when_content_too_short(monkeypatch):
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
return _FakeResponse(_FakeMessage('too short'))
|
||||
|
||||
monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient())
|
||||
|
||||
assert ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru') is None
|
||||
|
||||
|
||||
def test_fetch_with_ai_returns_none_on_exception(monkeypatch):
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
raise Exception('api down')
|
||||
|
||||
monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient())
|
||||
|
||||
assert ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru') is None
|
||||
|
||||
|
||||
def test_fetch_with_ai_appends_citation_content(monkeypatch):
|
||||
citation = type('Citation', (), {'content': 'extra citation text'})()
|
||||
annotation = type('Annotation', (), {'url_citation': citation})()
|
||||
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
return _FakeResponse(_FakeMessage('x' * 350, annotations=[annotation]))
|
||||
|
||||
monkeypatch.setattr(ai_fetch, 'OpenAI', lambda api_key, base_url: _FakeClient())
|
||||
|
||||
result = ai_fetch.fetch_with_ai('https://intrepidtravel.com/peru')
|
||||
assert 'extra citation text' in result['text']
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
import pytest
|
||||
from core import extractor
|
||||
from core.extractor import extract_with_openrouter, ExtractorError, RateLimitError, _parse_json_response
|
||||
|
||||
|
||||
def test_parse_json_response_handles_clean_json():
|
||||
result = _parse_json_response('{"tripName": "Peru Classic"}', 'model-a', 'test')
|
||||
assert result == {'tripName': 'Peru Classic'}
|
||||
|
||||
|
||||
def test_parse_json_response_strips_code_fences():
|
||||
result = _parse_json_response('```json\n{"tripName": "Peru Classic"}\n```', 'model-a', 'test')
|
||||
assert result == {'tripName': 'Peru Classic'}
|
||||
|
||||
|
||||
def test_parse_json_response_extracts_object_from_surrounding_text():
|
||||
result = _parse_json_response('Here is the data: {"tripName": "Peru Classic"} Thanks!', 'model-a', 'test')
|
||||
assert result == {'tripName': 'Peru Classic'}
|
||||
|
||||
|
||||
def test_parse_json_response_repairs_trailing_commas_and_unquoted_keys():
|
||||
malformed = '{tripName: "Peru Classic", duration: 15,}'
|
||||
result = _parse_json_response(malformed, 'model-a', 'test')
|
||||
assert result == {'tripName': 'Peru Classic', 'duration': 15}
|
||||
|
||||
|
||||
def test_parse_json_response_raises_when_unparseable():
|
||||
with pytest.raises(ValueError):
|
||||
_parse_json_response('not json at all !!!', 'model-a', 'test')
|
||||
|
||||
|
||||
class _FakeChoice:
|
||||
def __init__(self, content):
|
||||
self.message = type('Msg', (), {'content': content})()
|
||||
|
||||
|
||||
class _FakeCompletionResponse:
|
||||
def __init__(self, content):
|
||||
self.choices = [_FakeChoice(content)]
|
||||
|
||||
|
||||
def test_call_openrouter_raises_rate_limit_error(monkeypatch):
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
raise Exception('429 Too Many Requests: rate_limit_exceeded')
|
||||
|
||||
monkeypatch.setattr(extractor, 'OpenAI', lambda api_key, base_url: _FakeClient())
|
||||
with pytest.raises(RateLimitError):
|
||||
extractor._call_openrouter('some-model', 'prompt text', '')
|
||||
|
||||
|
||||
def test_call_openrouter_reraises_other_errors(monkeypatch):
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
raise Exception('500 server error')
|
||||
|
||||
monkeypatch.setattr(extractor, 'OpenAI', lambda api_key, base_url: _FakeClient())
|
||||
with pytest.raises(Exception, match='500 server error'):
|
||||
extractor._call_openrouter('some-model', 'prompt text', '')
|
||||
|
||||
|
||||
def test_call_openrouter_appends_content_when_present(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
captured['messages'] = kwargs['messages']
|
||||
return _FakeCompletionResponse('{"ok": true}')
|
||||
|
||||
monkeypatch.setattr(extractor, 'OpenAI', lambda api_key, base_url: _FakeClient())
|
||||
result = extractor._call_openrouter('some-model', 'BASE PROMPT', 'EXTRA CONTENT')
|
||||
|
||||
assert result == {'ok': True}
|
||||
assert 'BASE PROMPT' in captured['messages'][0]['content']
|
||||
assert 'EXTRA CONTENT' in captured['messages'][0]['content']
|
||||
|
||||
|
||||
def test_extract_with_openrouter_tries_next_model_on_rate_limit(monkeypatch):
|
||||
monkeypatch.setattr(extractor, 'FREE_MODELS', ['model-free-1', 'model-free-2'])
|
||||
monkeypatch.setattr(extractor, 'FALLBACK_MODEL', 'model-paid')
|
||||
|
||||
calls = []
|
||||
|
||||
def _fake_call(model, prompt, content):
|
||||
calls.append(model)
|
||||
if model == 'model-free-1':
|
||||
raise RateLimitError('rate limited')
|
||||
return {'tripName': 'ok'}
|
||||
|
||||
monkeypatch.setattr(extractor, '_call_openrouter', _fake_call)
|
||||
|
||||
result = extract_with_openrouter('prompt')
|
||||
|
||||
assert result == {'tripName': 'ok'}
|
||||
assert calls == ['model-free-1', 'model-free-2']
|
||||
|
||||
|
||||
def test_extract_with_openrouter_falls_back_to_paid_model_and_logs_sentry(monkeypatch):
|
||||
monkeypatch.setattr(extractor, 'FREE_MODELS', ['model-free-1'])
|
||||
monkeypatch.setattr(extractor, 'FALLBACK_MODEL', 'model-paid')
|
||||
|
||||
def _fake_call(model, prompt, content):
|
||||
if model == 'model-free-1':
|
||||
raise Exception('exhausted')
|
||||
return {'tripName': 'from-fallback'}
|
||||
|
||||
monkeypatch.setattr(extractor, '_call_openrouter', _fake_call)
|
||||
sentry_calls = []
|
||||
monkeypatch.setattr(extractor.sentry_sdk, 'capture_message', lambda msg, level=None: sentry_calls.append(msg))
|
||||
|
||||
result = extract_with_openrouter('prompt')
|
||||
|
||||
assert result == {'tripName': 'from-fallback'}
|
||||
assert sentry_calls == ['OpenRouter paid fallback used']
|
||||
|
||||
|
||||
def test_extract_with_openrouter_raises_when_all_models_exhausted(monkeypatch):
|
||||
monkeypatch.setattr(extractor, 'FREE_MODELS', ['model-free-1'])
|
||||
monkeypatch.setattr(extractor, 'FALLBACK_MODEL', 'model-paid')
|
||||
monkeypatch.setattr(extractor, '_call_openrouter', lambda *a, **k: (_ for _ in ()).throw(Exception('down')))
|
||||
|
||||
with pytest.raises(ExtractorError):
|
||||
extract_with_openrouter('prompt')
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
from core.proxy_service import ProxyService
|
||||
|
||||
|
||||
def test_webshare_requests_config_with_country():
|
||||
service = ProxyService()
|
||||
config = service._webshare_requests(country='gb')
|
||||
assert '-GB-rotate' in config['http']
|
||||
assert config['http'] == config['https']
|
||||
|
||||
|
||||
def test_webshare_requests_config_without_country():
|
||||
service = ProxyService()
|
||||
config = service._webshare_requests()
|
||||
assert '-rotate' in config['http']
|
||||
assert '-GB-' not in config['http']
|
||||
|
||||
|
||||
def test_webshare_playwright_config():
|
||||
service = ProxyService()
|
||||
config = service._webshare_playwright(country='us')
|
||||
assert config['server'] == 'http://p.webshare.io:80'
|
||||
assert config['username'].endswith('-US-rotate')
|
||||
|
||||
|
||||
def test_brightdata_requests_and_playwright_configs():
|
||||
service = ProxyService()
|
||||
assert service._brightdata_requests()['http'].startswith('http://')
|
||||
assert service._brightdata_playwright()['server'] == 'http://brd.superproxy.io:22225'
|
||||
|
||||
|
||||
def test_get_proxy_for_domain_uses_webshare_when_no_override(monkeypatch):
|
||||
service = ProxyService()
|
||||
monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None)
|
||||
config = service.get_proxy_for_domain('intrepidtravel.com', country='GB')
|
||||
assert 'p.webshare.io' in config['http']
|
||||
|
||||
|
||||
def test_get_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch):
|
||||
service = ProxyService()
|
||||
monkeypatch.setattr(
|
||||
'core.proxy_service.proxy_repository.get_by_domain',
|
||||
lambda domain: {'id': 'o1', 'provider': 'brightdata'}
|
||||
)
|
||||
config = service.get_proxy_for_domain('flashpack.com')
|
||||
assert 'brd.superproxy.io' in config['http']
|
||||
|
||||
|
||||
def test_get_playwright_proxy_for_domain_uses_brightdata_when_override_exists(monkeypatch):
|
||||
service = ProxyService()
|
||||
monkeypatch.setattr(
|
||||
'core.proxy_service.proxy_repository.get_by_domain',
|
||||
lambda domain: {'id': 'o1', 'provider': 'brightdata'}
|
||||
)
|
||||
config = service.get_playwright_proxy_for_domain('flashpack.com')
|
||||
assert config['server'] == 'http://brd.superproxy.io:22225'
|
||||
|
||||
|
||||
def test_escalate_to_brightdata_creates_new_override(monkeypatch):
|
||||
service = ProxyService()
|
||||
monkeypatch.setattr('core.proxy_service.proxy_repository.get_by_domain', lambda domain: None)
|
||||
created = {}
|
||||
monkeypatch.setattr('core.proxy_service.proxy_repository.create', lambda data: created.update(data))
|
||||
|
||||
service.escalate_to_brightdata('newlyblocked.com')
|
||||
|
||||
assert created['domain'] == 'newlyblocked.com'
|
||||
assert created['provider'] == 'brightdata'
|
||||
assert created['blocked_count'] == 1
|
||||
|
||||
|
||||
def test_escalate_to_brightdata_increments_existing_override(monkeypatch):
|
||||
service = ProxyService()
|
||||
monkeypatch.setattr(
|
||||
'core.proxy_service.proxy_repository.get_by_domain',
|
||||
lambda domain: {'id': 'o1', 'blocked_count': 2}
|
||||
)
|
||||
updated = {}
|
||||
monkeypatch.setattr(
|
||||
'core.proxy_service.proxy_repository.update',
|
||||
lambda override_id, data: updated.update({'id': override_id, **data})
|
||||
)
|
||||
|
||||
service.escalate_to_brightdata('repeatoffender.com')
|
||||
|
||||
assert updated['id'] == 'o1'
|
||||
assert updated['blocked_count'] == 3
|
||||
assert updated['provider'] == 'brightdata'
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
import asyncio
|
||||
from core import scraper
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
"""
|
||||
Runs a coroutine on a fresh, explicit event loop — mirrors exactly
|
||||
how core.scraper.scrape() drives render_page() in production.
|
||||
Deliberately NOT asyncio.run(): scrape()'s own nest_asyncio.apply()
|
||||
call (exercised by other tests in this file) patches asyncio's
|
||||
"current event loop" machinery process-wide, which makes a bare
|
||||
asyncio.run() here liable to pick up and try to reuse a closed loop
|
||||
left over from an earlier scrape() test, depending on test order.
|
||||
"""
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_extract_domain_strips_scheme_and_www():
|
||||
assert scraper.extract_domain('https://www.intrepidtravel.com/peru') == 'intrepidtravel.com'
|
||||
assert scraper.extract_domain('intrepidtravel.com') == 'intrepidtravel.com'
|
||||
|
||||
|
||||
def test_clean_url_strips_tracking_params():
|
||||
dirty = 'https://intrepidtravel.com/peru?utm_source=fb&utm_campaign=x&keep=1'
|
||||
clean = scraper.clean_url(dirty)
|
||||
assert 'utm_source' not in clean
|
||||
assert 'utm_campaign' not in clean
|
||||
assert 'keep=1' in clean
|
||||
|
||||
|
||||
def test_clean_url_strips_fragment():
|
||||
assert scraper.clean_url('https://intrepidtravel.com/peru#reviews') == 'https://intrepidtravel.com/peru'
|
||||
|
||||
|
||||
def test_clean_url_unchanged_when_already_clean():
|
||||
url = 'https://intrepidtravel.com/peru'
|
||||
assert scraper.clean_url(url) == url
|
||||
|
||||
|
||||
def test_is_blocked_true_on_low_char_count():
|
||||
assert scraper._is_blocked({'char_count': 100, 'text': ''}) is True
|
||||
|
||||
|
||||
def test_is_blocked_true_on_403_status():
|
||||
assert scraper._is_blocked({'char_count': 2000, 'status_code': 403, 'text': ''}) is True
|
||||
|
||||
|
||||
def test_is_blocked_true_on_captcha_text():
|
||||
assert scraper._is_blocked({'char_count': 2000, 'text': 'Please complete this CAPTCHA to continue'}) is True
|
||||
|
||||
|
||||
def test_is_blocked_false_for_normal_content():
|
||||
assert scraper._is_blocked({'char_count': 5000, 'status_code': 200, 'text': 'Peru Classic 15 days...'}) is False
|
||||
|
||||
|
||||
def test_check_robots_txt_permissive_default_on_fetch_failure(monkeypatch):
|
||||
class _RaisingParser:
|
||||
def set_url(self, url):
|
||||
pass
|
||||
|
||||
def read(self):
|
||||
raise Exception('unreachable')
|
||||
|
||||
monkeypatch.setattr(scraper, 'RobotFileParser', lambda: _RaisingParser())
|
||||
assert scraper.check_robots_txt('https://intrepidtravel.com/peru') is True
|
||||
|
||||
|
||||
def test_check_robots_txt_respects_disallow(monkeypatch):
|
||||
class _DisallowParser:
|
||||
def set_url(self, url):
|
||||
pass
|
||||
|
||||
def read(self):
|
||||
pass
|
||||
|
||||
def can_fetch(self, agent, url):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(scraper, 'RobotFileParser', lambda: _DisallowParser())
|
||||
assert scraper.check_robots_txt('https://intrepidtravel.com/peru') is False
|
||||
|
||||
|
||||
def test_scrape_skips_when_robots_txt_disallows(monkeypatch):
|
||||
monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: False)
|
||||
result = scraper.scrape('https://intrepidtravel.com/peru')
|
||||
assert result['success'] is False
|
||||
assert 'robots.txt' in result['text']
|
||||
|
||||
|
||||
def test_scrape_returns_playwright_result_when_content_is_sufficient(monkeypatch):
|
||||
monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: True)
|
||||
|
||||
async def _fake_render_page(url, country=None):
|
||||
return {'text': 'x' * 2000, 'tables': '', 'url': url, 'char_count': 2000, 'success': True}
|
||||
|
||||
monkeypatch.setattr(scraper, 'render_page', _fake_render_page)
|
||||
|
||||
result = scraper.scrape('https://intrepidtravel.com/peru')
|
||||
assert result['char_count'] == 2000
|
||||
assert result['success'] is True
|
||||
|
||||
|
||||
def test_scrape_falls_back_to_ai_fetch_when_char_count_low(monkeypatch):
|
||||
import sys
|
||||
import types
|
||||
|
||||
monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: True)
|
||||
|
||||
async def _fake_render_page(url, country=None):
|
||||
return {'text': 'short', 'tables': '', 'url': url, 'char_count': 5, 'success': True}
|
||||
|
||||
monkeypatch.setattr(scraper, 'render_page', _fake_render_page)
|
||||
|
||||
fake_ai_fetch = types.ModuleType('core.ai_fetch')
|
||||
fake_ai_fetch.fetch_with_ai = lambda url: {
|
||||
'text': 'ai-fetched content', 'tables': '', 'url': url, 'char_count': 2000, 'success': True
|
||||
}
|
||||
monkeypatch.setitem(sys.modules, 'core.ai_fetch', fake_ai_fetch)
|
||||
|
||||
result = scraper.scrape('https://intrepidtravel.com/peru')
|
||||
assert result['text'] == 'ai-fetched content'
|
||||
|
||||
|
||||
def test_render_page_uses_webshare_proxy_when_not_blocked(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
scraper.proxy_service, 'get_playwright_proxy_for_domain',
|
||||
lambda domain, country=None: {'server': 'webshare-proxy'}
|
||||
)
|
||||
calls = []
|
||||
|
||||
async def _fake_render_with_proxy(url, proxy):
|
||||
calls.append(proxy)
|
||||
return {'char_count': 5000, 'status_code': 200, 'text': 'good content', 'success': True, 'url': url}
|
||||
|
||||
monkeypatch.setattr(scraper, '_render_with_proxy', _fake_render_with_proxy)
|
||||
escalated = []
|
||||
monkeypatch.setattr(scraper.proxy_service, 'escalate_to_brightdata', lambda domain: escalated.append(domain))
|
||||
|
||||
result = _run_async(scraper.render_page('https://intrepidtravel.com/peru', country='GB'))
|
||||
|
||||
assert result['success'] is True
|
||||
assert calls == [{'server': 'webshare-proxy'}]
|
||||
assert escalated == []
|
||||
|
||||
|
||||
def test_render_page_escalates_to_brightdata_when_blocked(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
scraper.proxy_service, 'get_playwright_proxy_for_domain',
|
||||
lambda domain, country=None: {'server': 'webshare-proxy'}
|
||||
)
|
||||
monkeypatch.setattr(scraper.proxy_service, '_brightdata_playwright', lambda: {'server': 'brightdata-proxy'})
|
||||
|
||||
call_log = []
|
||||
|
||||
async def _fake_render_with_proxy(url, proxy):
|
||||
call_log.append(proxy)
|
||||
if proxy['server'] == 'webshare-proxy':
|
||||
return {'char_count': 50, 'status_code': 403, 'text': 'blocked', 'success': True, 'url': url}
|
||||
return {'char_count': 5000, 'status_code': 200, 'text': 'good content', 'success': True, 'url': url}
|
||||
|
||||
monkeypatch.setattr(scraper, '_render_with_proxy', _fake_render_with_proxy)
|
||||
escalated = []
|
||||
monkeypatch.setattr(scraper.proxy_service, 'escalate_to_brightdata', lambda domain: escalated.append(domain))
|
||||
|
||||
result = _run_async(scraper.render_page('https://intrepidtravel.com/peru', country='GB'))
|
||||
|
||||
assert result['text'] == 'good content'
|
||||
assert escalated == ['intrepidtravel.com']
|
||||
assert call_log == [{'server': 'webshare-proxy'}, {'server': 'brightdata-proxy'}]
|
||||
|
||||
|
||||
def test_scrape_keeps_playwright_result_when_ai_fetch_returns_none(monkeypatch):
|
||||
import sys
|
||||
import types
|
||||
|
||||
monkeypatch.setattr(scraper, 'check_robots_txt', lambda url: True)
|
||||
|
||||
async def _fake_render_page(url, country=None):
|
||||
return {'text': 'short', 'tables': '', 'url': url, 'char_count': 5, 'success': True}
|
||||
|
||||
monkeypatch.setattr(scraper, 'render_page', _fake_render_page)
|
||||
|
||||
fake_ai_fetch = types.ModuleType('core.ai_fetch')
|
||||
fake_ai_fetch.fetch_with_ai = lambda url: None
|
||||
monkeypatch.setitem(sys.modules, 'core.ai_fetch', fake_ai_fetch)
|
||||
|
||||
result = scraper.scrape('https://intrepidtravel.com/peru')
|
||||
assert result['char_count'] == 5
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
"""
|
||||
analysis_service orchestrates the fast/refresh path decision (CLAUDE.md
|
||||
§7). The refresh path lazy-imports Phase 2 modules (core.scraper,
|
||||
core.extractor, industries.adventure_travel.prompt) that don't exist
|
||||
yet — _install_fake_module injects stand-ins into sys.modules so the
|
||||
orchestration logic itself (not the scraping/extraction internals) can
|
||||
be tested now.
|
||||
§7). It calls into core.scraper / core.extractor /
|
||||
industries.adventure_travel.prompt, all of which are now real Phase 2
|
||||
modules — _install_fake_module swaps out just the functions this test
|
||||
needs via monkeypatch.setattr (properly reverted after each test),
|
||||
rather than mutating the shared module objects directly.
|
||||
"""
|
||||
import sys
|
||||
import types
|
||||
|
|
@ -17,7 +17,15 @@ from repositories.tenant_repository import tenant_repository
|
|||
|
||||
|
||||
def _install_fake_module(monkeypatch, dotted_name, **attrs):
|
||||
"""Injects a fake module into sys.modules, creating parent packages as needed."""
|
||||
"""
|
||||
Ensures `dotted_name` resolves to a module (creating fake parent
|
||||
packages only if truly absent), then uses monkeypatch.setattr for
|
||||
every attribute — critical when the target is a REAL module (as
|
||||
core.scraper/core.extractor/industries.adventure_travel.prompt all
|
||||
are post-Phase-2): monkeypatch.setattr records the original value
|
||||
and restores it on teardown, whereas a raw setattr would silently
|
||||
and permanently corrupt the shared module for every later test.
|
||||
"""
|
||||
parts = dotted_name.split('.')
|
||||
for i in range(1, len(parts) + 1):
|
||||
partial = '.'.join(parts[:i])
|
||||
|
|
@ -25,7 +33,7 @@ def _install_fake_module(monkeypatch, dotted_name, **attrs):
|
|||
monkeypatch.setitem(sys.modules, partial, types.ModuleType(partial))
|
||||
target = sys.modules[dotted_name]
|
||||
for key, value in attrs.items():
|
||||
setattr(target, key, value)
|
||||
monkeypatch.setattr(target, key, value)
|
||||
return target
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@ from repositories.battlecard_repository import battlecard_repository
|
|||
|
||||
|
||||
def _install_fake_extractor(monkeypatch, fn):
|
||||
# core.extractor is now a real Phase 2 module — must use
|
||||
# monkeypatch.setattr (not a raw setattr) so the swap reverts after
|
||||
# the test instead of permanently corrupting the shared module.
|
||||
for name in ('core', 'core.extractor'):
|
||||
if name not in sys.modules:
|
||||
monkeypatch.setitem(sys.modules, name, types.ModuleType(name))
|
||||
sys.modules['core.extractor'].extract_with_openrouter = fn
|
||||
monkeypatch.setattr(sys.modules['core.extractor'], 'extract_with_openrouter', fn)
|
||||
|
||||
|
||||
def _setup():
|
||||
|
|
|
|||
|
|
@ -25,13 +25,20 @@ def _make_tenant(**overrides):
|
|||
|
||||
|
||||
def _install_fake_module(monkeypatch, dotted_name, **attrs):
|
||||
"""
|
||||
Ensures `dotted_name` resolves to a module, then uses
|
||||
monkeypatch.setattr for every attribute — required now that
|
||||
core.extractor/industries.adventure_travel.prompt are real Phase 2
|
||||
modules: a raw setattr would permanently corrupt the shared module
|
||||
for every later test since monkeypatch never recorded it.
|
||||
"""
|
||||
parts = dotted_name.split('.')
|
||||
for i in range(1, len(parts) + 1):
|
||||
partial = '.'.join(parts[:i])
|
||||
if partial not in sys.modules:
|
||||
monkeypatch.setitem(sys.modules, partial, types.ModuleType(partial))
|
||||
for key, value in attrs.items():
|
||||
setattr(sys.modules[dotted_name], key, value)
|
||||
monkeypatch.setattr(sys.modules[dotted_name], key, value)
|
||||
|
||||
|
||||
# ── Account ────────────────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
from config import client as config_client
|
||||
|
||||
|
||||
class _FakeWorksheet:
|
||||
def __init__(self, records=None):
|
||||
self._records = records or []
|
||||
self.batch_updates = []
|
||||
self.cell_updates = []
|
||||
|
||||
def batch_update(self, updates):
|
||||
self.batch_updates.append(updates)
|
||||
|
||||
def get_all_records(self):
|
||||
return self._records
|
||||
|
||||
def update_cell(self, row, col, value):
|
||||
self.cell_updates.append((row, col, value))
|
||||
|
||||
|
||||
class _FakeSheet:
|
||||
def __init__(self, worksheets):
|
||||
self._worksheets = worksheets
|
||||
|
||||
def worksheet(self, name):
|
||||
return self._worksheets[name]
|
||||
|
||||
|
||||
class _FakeGspreadClient:
|
||||
def __init__(self, sheet):
|
||||
self._sheet = sheet
|
||||
|
||||
def open_by_key(self, sheet_id):
|
||||
return self._sheet
|
||||
|
||||
|
||||
def test_write_to_sheet_only_writes_non_empty_fields(monkeypatch):
|
||||
ws = _FakeWorksheet()
|
||||
fake_sheet = _FakeSheet({'Standard Competitive Tab': ws})
|
||||
monkeypatch.setattr(config_client, 'get_sheet_client', lambda: _FakeGspreadClient(fake_sheet))
|
||||
|
||||
config_client.write_to_sheet(
|
||||
'Standard Competitive Tab', col_index=4,
|
||||
data={'tripName': 'Peru Classic', 'tripCode': None, 'majorityPrice': ''},
|
||||
fields={'tripName': 9, 'tripCode': 10, 'majorityPrice': 25}
|
||||
)
|
||||
|
||||
assert len(ws.batch_updates) == 1
|
||||
updates = ws.batch_updates[0]
|
||||
assert len(updates) == 1 # only tripName had a non-empty value
|
||||
assert updates[0]['values'] == [['Peru Classic']]
|
||||
|
||||
|
||||
def test_write_to_sheet_skips_batch_update_when_nothing_to_write(monkeypatch):
|
||||
ws = _FakeWorksheet()
|
||||
fake_sheet = _FakeSheet({'Standard Competitive Tab': ws})
|
||||
monkeypatch.setattr(config_client, 'get_sheet_client', lambda: _FakeGspreadClient(fake_sheet))
|
||||
|
||||
config_client.write_to_sheet(
|
||||
'Standard Competitive Tab', col_index=4, data={'tripName': None}, fields={'tripName': 9}
|
||||
)
|
||||
|
||||
assert ws.batch_updates == []
|
||||
|
||||
|
||||
def test_get_competitors_from_db_filters_active_only(monkeypatch):
|
||||
ws = _FakeWorksheet(records=[
|
||||
{'Travel Style': 'Classic', 'Competitor Name': 'Intrepid', 'Website URL': 'https://intrepidtravel.com', 'Active': 'Yes'},
|
||||
{'Travel Style': 'Premium', 'Competitor Name': 'Flash Pack', 'Website URL': 'https://flashpack.com', 'Active': 'No'},
|
||||
])
|
||||
fake_sheet = _FakeSheet({'Competitor DB': ws})
|
||||
monkeypatch.setattr(config_client, 'get_sheet_client', lambda: _FakeGspreadClient(fake_sheet))
|
||||
|
||||
result = config_client.get_competitors_from_db()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]['name'] == 'Intrepid'
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
from industries.adventure_travel import fields, prompt, schema
|
||||
|
||||
|
||||
def test_field_maps_have_matching_keys_for_shared_fields():
|
||||
shared_keys = set(fields.STANDARD_FIELDS) & set(fields.NGS_FIELDS)
|
||||
assert 'tripName' in shared_keys
|
||||
assert 'majorityPrice' in shared_keys
|
||||
assert 'exclusiveAccess' not in fields.STANDARD_FIELDS
|
||||
assert 'exclusiveAccess' in fields.NGS_FIELDS
|
||||
|
||||
|
||||
def test_build_prompt_embeds_page_content_and_context():
|
||||
page_data = {'text': 'Peru Classic 15 days itinerary...', 'tables': 'Day | Activity', 'url': 'https://intrepidtravel.com/peru'}
|
||||
result = prompt.build_prompt('Intrepid Travel', page_data, 'Inca Trail', 'Peru', 15, 'Classic')
|
||||
|
||||
assert 'Intrepid Travel' in result
|
||||
assert 'Peru Classic 15 days itinerary' in result
|
||||
assert 'Day | Activity' in result
|
||||
assert 'Peru' in result
|
||||
assert '"exclusiveAccess"' not in result
|
||||
|
||||
|
||||
def test_build_prompt_includes_ngs_fields_when_flagged():
|
||||
page_data = {'text': 'content', 'tables': '', 'url': 'https://example.com'}
|
||||
result = prompt.build_prompt('Competitor', page_data, 'Product', 'Peru', 10, 'NGS', is_ngs=True)
|
||||
assert '"exclusiveAccess"' in result
|
||||
assert '"groupLeader"' in result
|
||||
assert '"sustainability"' in result
|
||||
|
||||
|
||||
def test_build_prompt_handles_missing_page_data():
|
||||
result = prompt.build_prompt('Competitor', None, 'Product', 'Peru', 10, 'Classic')
|
||||
assert '[PAGE CONTENT WILL BE INSERTED HERE]' in result
|
||||
assert '[COMPETITOR URL]' in result
|
||||
|
||||
|
||||
def test_inject_page_content_replaces_placeholders():
|
||||
template = (
|
||||
'Scraped from Competitor tour page at: [COMPETITOR URL]\n'
|
||||
'PAGE CONTENT:\n[PAGE CONTENT WILL BE INSERTED HERE]'
|
||||
)
|
||||
page_data = {'text': 'Real scraped content', 'tables': '', 'url': 'https://intrepidtravel.com/peru'}
|
||||
|
||||
result = prompt.inject_page_content(template, 'Competitor', page_data)
|
||||
|
||||
assert 'Real scraped content' in result
|
||||
assert 'https://intrepidtravel.com/peru' in result
|
||||
assert '[COMPETITOR URL]' not in result
|
||||
|
||||
|
||||
def test_inject_page_content_appends_table_data():
|
||||
template = 'PAGE CONTENT:\n[PAGE CONTENT WILL BE INSERTED HERE]\nat [COMPETITOR URL]'
|
||||
page_data = {'text': 'Base text', 'tables': 'Day | Meal', 'url': 'https://example.com'}
|
||||
|
||||
result = prompt.inject_page_content(template, 'Competitor', page_data)
|
||||
|
||||
assert 'TABLE DATA' in result
|
||||
assert 'Day | Meal' in result
|
||||
|
||||
|
||||
def test_ngs_schema_extends_standard_schema():
|
||||
assert set(schema.STANDARD_SCHEMA).issubset(set(schema.NGS_SCHEMA))
|
||||
assert 'exclusiveAccess' in schema.NGS_SCHEMA
|
||||
assert 'exclusiveAccess' not in schema.STANDARD_SCHEMA
|
||||
Loading…
Reference in New Issue