109 lines
4.9 KiB
Python
109 lines
4.9 KiB
Python
import os
|
|
import logging
|
|
from urllib.parse import urlparse
|
|
from openai import OpenAI
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
|
|
|
|
# Note: this fallback calls OpenRouter's server-side web_search tool
|
|
# (Exa engine) rather than making a direct HTTP request to the
|
|
# competitor's domain — the search itself runs on OpenRouter's
|
|
# infrastructure, not ours, so core/proxy_service.py's Webshare/Bright
|
|
# Data routing has no attachment point here. Ported unchanged from the
|
|
# legacy app.py per CLAUDE.md §10 ("logic stays identical").
|
|
|
|
|
|
def fetch_with_ai(url):
|
|
"""
|
|
AI web fetch fallback for bot-protected sites. Fires only when
|
|
Playwright returns under 1000 chars. Uses OpenRouter's web_search
|
|
server tool with the Exa engine, restricted to the competitor's
|
|
own domain. Cost: ~$0.02/call + tokens — only fires for
|
|
bot-protected sites, not every scrape.
|
|
|
|
The returned text feeds directly into build_prompt()'s PAGE CONTENT
|
|
block, which is written for plain scraped text describing one trip
|
|
— the prompt below explicitly asks for plain text (no markdown) and
|
|
a single trip, matching that shape. Discovered why this matters via
|
|
a real failure: gpt-4o-mini's *default* style for "search and
|
|
summarize" is markdown — headers, bold labels, numbered lists of
|
|
every trip it found across the domain. That got embedded into the
|
|
extraction prompt's "Return ONLY a valid JSON object, no markdown"
|
|
instruction, and the extraction model — confused about which of
|
|
several listed trips to extract, and primed by markdown-formatted
|
|
input — produced unparseable output. This is a pre-existing
|
|
condition (ported unchanged from the legacy app.py, which has the
|
|
identical prompt), not something the modular refactor introduced.
|
|
|
|
Data flow:
|
|
url → domain extracted → OpenRouter chat completion with
|
|
web_search tool (allowed_domains=[domain]) →
|
|
response text + any citation content →
|
|
{ text, tables: '', url, char_count, success } or None if the
|
|
fallback also returns too little content
|
|
"""
|
|
logger.info(f'Playwright failed — attempting AI web fetch for {url}')
|
|
try:
|
|
client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1')
|
|
domain = urlparse(url).netloc
|
|
|
|
response = client.chat.completions.create(
|
|
model='openai/gpt-4o-mini',
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': (
|
|
'I am a travel industry analyst conducting competitive research. '
|
|
'Search specifically for the single tour page at this exact URL — '
|
|
'not other tours or pages on this website.\n\n'
|
|
f'URL: {url}\n\n'
|
|
'Extract only the factual data points for THIS ONE tour: '
|
|
'trip name, trip code, price, duration in days, maximum group size, '
|
|
'total included meals, hotel names, departure dates, start location, '
|
|
'end location, service level, and key activities.\n\n'
|
|
'Respond in plain text only — no markdown, no headers, no bold text, '
|
|
'no bullet points, no numbered lists. Write each fact as a short '
|
|
'"label: value" line, one per line, the way you would plainly summarise '
|
|
'a single web page. If a fact is not found, write "not found" for that '
|
|
'field rather than omitting it.'
|
|
)
|
|
}],
|
|
tools=[{
|
|
'type': 'openrouter:web_search',
|
|
'parameters': {
|
|
'engine': 'exa',
|
|
'max_results': 5,
|
|
'search_context_size': 'high',
|
|
'allowed_domains': [domain],
|
|
}
|
|
}],
|
|
temperature=0.0,
|
|
max_tokens=8192
|
|
)
|
|
|
|
msg = response.choices[0].message
|
|
text = msg.content if isinstance(msg.content, str) else ''
|
|
|
|
# Also pull content from tool-result annotations if present
|
|
if hasattr(msg, 'annotations') and msg.annotations:
|
|
for annotation in msg.annotations:
|
|
if hasattr(annotation, 'url_citation') and annotation.url_citation:
|
|
citation_content = getattr(annotation.url_citation, 'content', '')
|
|
if citation_content:
|
|
text = text + '\n\n' + citation_content
|
|
|
|
text = text.strip()
|
|
char_count = len(text)
|
|
logger.info(f'AI web fetch returned {char_count} chars for {url}')
|
|
|
|
if char_count > 300:
|
|
return {'text': text[:25000], 'tables': '', 'url': url, 'char_count': char_count, 'success': True}
|
|
|
|
logger.warning(f'AI web fetch also returned low content for {url}')
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f'AI web fetch failed for {url}: {str(e)}')
|
|
return None
|