88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
import os
|
|
import logging
|
|
from urllib.parse import urlparse
|
|
from openai import OpenAI
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
|
|
|
|
# Note: this fallback calls OpenRouter's server-side web_search tool
|
|
# (Exa engine) rather than making a direct HTTP request to the
|
|
# competitor's domain — the search itself runs on OpenRouter's
|
|
# infrastructure, not ours, so core/proxy_service.py's Webshare/Bright
|
|
# Data routing has no attachment point here. Ported unchanged from the
|
|
# legacy app.py per CLAUDE.md §10 ("logic stays identical").
|
|
|
|
|
|
def fetch_with_ai(url):
|
|
"""
|
|
AI web fetch fallback for bot-protected sites. Fires only when
|
|
Playwright returns under 1000 chars. Uses OpenRouter's web_search
|
|
server tool with the Exa engine, restricted to the competitor's
|
|
own domain. Cost: ~$0.02/call + tokens — only fires for
|
|
bot-protected sites, not every scrape.
|
|
|
|
Data flow:
|
|
url → domain extracted → OpenRouter chat completion with
|
|
web_search tool (allowed_domains=[domain]) →
|
|
response text + any citation content →
|
|
{ text, tables: '', url, char_count, success } or None if the
|
|
fallback also returns too little content
|
|
"""
|
|
logger.info(f'Playwright failed — attempting AI web fetch for {url}')
|
|
try:
|
|
client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1')
|
|
domain = urlparse(url).netloc
|
|
|
|
response = client.chat.completions.create(
|
|
model='openai/gpt-4o-mini',
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': (
|
|
'I am a travel industry analyst conducting competitive research. '
|
|
'Please search for this tour page and extract the following factual data points: '
|
|
'trip name, trip code, price, duration in days, maximum group size, '
|
|
'total included meals, hotel names, departure dates, start location, '
|
|
'end location, service level, and key activities. '
|
|
f'Return all factual information you find.\n\nURL: {url}'
|
|
)
|
|
}],
|
|
tools=[{
|
|
'type': 'openrouter:web_search',
|
|
'parameters': {
|
|
'engine': 'exa',
|
|
'max_results': 5,
|
|
'search_context_size': 'high',
|
|
'allowed_domains': [domain],
|
|
}
|
|
}],
|
|
temperature=0.0,
|
|
max_tokens=8192
|
|
)
|
|
|
|
msg = response.choices[0].message
|
|
text = msg.content if isinstance(msg.content, str) else ''
|
|
|
|
# Also pull content from tool-result annotations if present
|
|
if hasattr(msg, 'annotations') and msg.annotations:
|
|
for annotation in msg.annotations:
|
|
if hasattr(annotation, 'url_citation') and annotation.url_citation:
|
|
citation_content = getattr(annotation.url_citation, 'content', '')
|
|
if citation_content:
|
|
text = text + '\n\n' + citation_content
|
|
|
|
text = text.strip()
|
|
char_count = len(text)
|
|
logger.info(f'AI web fetch returned {char_count} chars for {url}')
|
|
|
|
if char_count > 300:
|
|
return {'text': text[:25000], 'tables': '', 'url': url, 'char_count': char_count, 'success': True}
|
|
|
|
logger.warning(f'AI web fetch also returned low content for {url}')
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f'AI web fetch failed for {url}: {str(e)}')
|
|
return None
|