764 lines
30 KiB
Python
764 lines
30 KiB
Python
import os
|
|
import json
|
|
import re
|
|
import asyncio
|
|
import time
|
|
import logging
|
|
from flask import Flask, request, jsonify
|
|
from dotenv import load_dotenv
|
|
from playwright.async_api import async_playwright
|
|
from bs4 import BeautifulSoup
|
|
# import google.generativeai as genai # ── Uncomment to use Gemini instead of OpenRouter
|
|
import gspread
|
|
from openai import OpenAI
|
|
from google.oauth2.service_account import Credentials
|
|
|
|
load_dotenv()
|
|
|
|
app = Flask(__name__)
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ── CONFIG ───────────────────────────────────────────────────
|
|
|
|
# GEMINI_API_KEY = os.getenv('GEMINI_API_KEY') # ── Uncomment to use Gemini
|
|
GROQ_API_KEY = os.getenv('GROQ_API_KEY')
|
|
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
|
|
API_SECRET_KEY = os.getenv('API_SECRET_KEY')
|
|
SHEET_ID = os.getenv('SHEET_ID')
|
|
STANDARD_TAB = os.getenv('STANDARD_TAB', 'Standard Competitive Tab')
|
|
NGS_TAB = os.getenv('NGS_TAB', 'NGS Competitive Tab')
|
|
SERVICE_ACCOUNT_JSON = os.getenv('GOOGLE_SERVICE_ACCOUNT_JSON')
|
|
|
|
# ── FIELD MAPS ───────────────────────────────────────────────
|
|
|
|
STANDARD_FIELDS = {
|
|
'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
|
|
'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
|
|
'startLocation': 15, 'endLocation': 16, 'departures': 17,
|
|
'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
|
|
'hotels': 22, 'activities': 23, 'relevancy': 24,
|
|
'majorityPrice': 25, 'priceLow': 26, 'priceHigh': 27,
|
|
'comments': 30, 'dateUsed': 31,
|
|
'audPrice': 32, 'cadPrice': 33, 'eurPrice': 34, 'gbpPrice': 35
|
|
}
|
|
|
|
NGS_FIELDS = {
|
|
'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
|
|
'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
|
|
'startLocation': 15, 'endLocation': 16, 'departures': 17,
|
|
'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
|
|
'hotels': 22, 'activities': 23, 'exclusiveAccess': 24,
|
|
'groupLeader': 25, 'sustainability': 26, 'relevancy': 27,
|
|
'majorityPrice': 28, 'priceLow': 29, 'priceHigh': 30,
|
|
'comments': 33, 'dateUsed': 34,
|
|
'audPrice': 35, 'cadPrice': 36, 'eurPrice': 37, 'gbpPrice': 38
|
|
}
|
|
|
|
|
|
# ── AUTH ─────────────────────────────────────────────────────
|
|
|
|
def check_auth(req):
|
|
token = req.headers.get('X-API-Key')
|
|
if not token or token != API_SECRET_KEY:
|
|
return False
|
|
return True
|
|
|
|
|
|
# ── GOOGLE SHEETS ────────────────────────────────────────────
|
|
|
|
def get_sheet_client():
|
|
scopes = [
|
|
'https://www.googleapis.com/auth/spreadsheets',
|
|
'https://www.googleapis.com/auth/drive'
|
|
]
|
|
info = json.loads(SERVICE_ACCOUNT_JSON)
|
|
creds = Credentials.from_service_account_info(info, scopes=scopes)
|
|
return gspread.authorize(creds)
|
|
|
|
|
|
def write_to_sheet(tab_name, col_index, data, fields):
|
|
client = get_sheet_client()
|
|
sheet = client.open_by_key(SHEET_ID)
|
|
ws = sheet.worksheet(tab_name)
|
|
updates = []
|
|
for field, row in fields.items():
|
|
value = data.get(field)
|
|
if value is None or value == '':
|
|
continue
|
|
updates.append({
|
|
'range': gspread.utils.rowcol_to_a1(row, col_index),
|
|
'values': [[value]]
|
|
})
|
|
if updates:
|
|
ws.batch_update(updates)
|
|
|
|
|
|
def get_competitors_from_db():
|
|
client = get_sheet_client()
|
|
sheet = client.open_by_key(SHEET_ID)
|
|
ws = sheet.worksheet('Competitor DB')
|
|
rows = ws.get_all_records()
|
|
return [
|
|
{
|
|
'style': r['Travel Style'],
|
|
'name': r['Competitor Name'],
|
|
'url': r['Website URL'],
|
|
'active': r['Active']
|
|
}
|
|
for r in rows
|
|
if str(r.get('Active', '')).strip().lower() == 'yes'
|
|
]
|
|
|
|
|
|
# ── PLAYWRIGHT SCRAPER ───────────────────────────────────────
|
|
|
|
async def render_page(url: str) -> dict:
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(
|
|
headless=True,
|
|
args=['--no-sandbox', '--disable-setuid-sandbox',
|
|
'--disable-dev-shm-usage', '--disable-gpu']
|
|
)
|
|
context = await browser.new_context(
|
|
user_agent=(
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
|
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
|
'Chrome/124.0.0.0 Safari/537.36'
|
|
),
|
|
viewport={'width': 1366, 'height': 768},
|
|
locale='en-GB',
|
|
timezone_id='Europe/London',
|
|
extra_http_headers={
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-GB,en;q=0.9',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'DNT': '1',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
'Sec-Fetch-Dest': 'document',
|
|
'Sec-Fetch-Mode': 'navigate',
|
|
'Sec-Fetch-Site': 'none',
|
|
'Sec-Fetch-User': '?1',
|
|
}
|
|
)
|
|
page = await context.new_page()
|
|
|
|
# Basic stealth — manual navigator spoofing
|
|
await page.add_init_script("""
|
|
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
|
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
|
|
Object.defineProperty(navigator, 'languages', { get: () => ['en-GB', 'en'] });
|
|
window.chrome = { runtime: {} };
|
|
""")
|
|
|
|
# Block only known third-party analytics/ad domains
|
|
BLOCKED_DOMAINS = [
|
|
'google-analytics.com', 'googletagmanager.com', 'doubleclick.net',
|
|
'facebook.net', 'facebook.com/tr', 'hotjar.com', 'intercom.io',
|
|
'zendesk.com', 'hubspot.com', 'marketo.com', 'segment.com',
|
|
'amplitude.com', 'mixpanel.com', 'optimizely.com', 'crazyegg.com',
|
|
]
|
|
|
|
async def block_trackers(route):
|
|
if any(domain in route.request.url for domain in BLOCKED_DOMAINS):
|
|
await route.abort()
|
|
else:
|
|
await route.continue_()
|
|
|
|
await page.route('**/*', block_trackers)
|
|
|
|
try:
|
|
try:
|
|
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
|
logger.info(f'Loaded: {url}')
|
|
except Exception as e:
|
|
logger.warning(f'domcontentloaded failed, trying networkidle: {str(e)[:100]}')
|
|
await page.goto(url, wait_until='networkidle', timeout=60000)
|
|
|
|
# Dismiss cookie consent banners
|
|
try:
|
|
await page.click(
|
|
'button:has-text("Accept All"), button:has-text("Accept all"), '
|
|
'button:has-text("Accept"), button:has-text("OK"), '
|
|
'button:has-text("I agree"), button:has-text("Agree"), '
|
|
'[id*="cookie"] button, [class*="cookie"] button, '
|
|
'[id*="consent"] button, [class*="consent"] button',
|
|
timeout=3000
|
|
)
|
|
logger.info('Cookie banner dismissed')
|
|
await page.wait_for_timeout(1000)
|
|
except Exception:
|
|
pass
|
|
|
|
# Wait for meaningful content
|
|
try:
|
|
await page.wait_for_selector(
|
|
'h1, .trip-overview, main, article, #overview, '
|
|
'[class*="itinerary"], [class*="price"], [class*="trip"]',
|
|
timeout=8000
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
await page.wait_for_timeout(3000)
|
|
|
|
# Two scroll passes to trigger lazy-loaded content
|
|
for _ in range(2):
|
|
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
|
await page.wait_for_timeout(1500)
|
|
await page.evaluate('window.scrollTo(0, 0)')
|
|
await page.wait_for_timeout(500)
|
|
|
|
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
|
await page.wait_for_timeout(2000)
|
|
|
|
html = await page.content()
|
|
soup = BeautifulSoup(html, 'lxml')
|
|
|
|
for tag in soup(['script', 'style', 'nav', 'footer',
|
|
'header', 'iframe', 'noscript', 'svg',
|
|
'button', 'form']):
|
|
tag.decompose()
|
|
|
|
# Prioritise main content area
|
|
main_content = (
|
|
soup.find('main') or
|
|
soup.find('article') or
|
|
soup.find(id='main') or
|
|
soup.find(id='content') or
|
|
soup.find(class_='content') or
|
|
soup
|
|
)
|
|
|
|
text = main_content.get_text(separator=' ', strip=True)
|
|
text = re.sub(r'\s+', ' ', text).strip()
|
|
|
|
tables = [t.get_text(separator=' | ', strip=True)
|
|
for t in soup.find_all('table')]
|
|
table_text = '\n'.join(tables)
|
|
char_count = len(text)
|
|
|
|
if char_count < 500:
|
|
logger.warning(
|
|
f'Low char count ({char_count}) for {url} — '
|
|
f'will attempt AI web fetch fallback'
|
|
)
|
|
|
|
return {
|
|
'text': text[:25000],
|
|
'tables': table_text[:5000],
|
|
'url': url,
|
|
'char_count': char_count,
|
|
'success': True
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f'Render failed for {url}: {str(e)}')
|
|
return {
|
|
'text': f'Render failed: {str(e)}',
|
|
'tables': '', 'url': url,
|
|
'char_count': 0, 'success': False
|
|
}
|
|
finally:
|
|
await browser.close()
|
|
|
|
|
|
def clean_url(url: str) -> str:
|
|
"""Strip URL fragments and common tracking parameters."""
|
|
from urllib.parse import urlparse, urlencode, parse_qs, urlunparse
|
|
parsed = urlparse(url)
|
|
STRIP_PARAMS = {
|
|
'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
|
|
'referrerView', 'ref', 'referrer', 'source', 'fbclid', 'gclid',
|
|
'mc_cid', 'mc_eid', '_ga', 'affiliate'
|
|
}
|
|
if parsed.query:
|
|
params = parse_qs(parsed.query, keep_blank_values=True)
|
|
cleaned = {k: v for k, v in params.items() if k not in STRIP_PARAMS}
|
|
query = urlencode(cleaned, doseq=True)
|
|
else:
|
|
query = ''
|
|
clean = urlunparse((
|
|
parsed.scheme, parsed.netloc, parsed.path,
|
|
parsed.params, query, ''
|
|
))
|
|
if clean != url:
|
|
logger.info(f'URL cleaned: {url} → {clean}')
|
|
return clean
|
|
|
|
|
|
def scrape(url):
|
|
import nest_asyncio
|
|
nest_asyncio.apply()
|
|
url = clean_url(url)
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
try:
|
|
result = loop.run_until_complete(render_page(url))
|
|
finally:
|
|
loop.close()
|
|
|
|
# AI web fetch fallback for bot-protected sites
|
|
if result.get('char_count', 0) < 1000 and result.get('success'):
|
|
logger.info(f'Playwright char count low — trying AI web fetch for {url}')
|
|
ai_result = fetch_with_ai(url)
|
|
if ai_result:
|
|
return ai_result
|
|
|
|
return result
|
|
|
|
|
|
# ── AI WEB FETCH FALLBACK ────────────────────────────────────
|
|
# Fires only when Playwright returns < 500 chars.
|
|
# Uses OpenRouter web_search server tool with Exa engine.
|
|
# Cost: ~$0.02 per call + token cost. Only fires for bot-protected sites.
|
|
|
|
def fetch_with_ai(url: str) -> dict:
|
|
logger.info(f'Playwright failed — attempting AI web fetch for {url}')
|
|
try:
|
|
client = OpenAI(
|
|
api_key=OPENROUTER_API_KEY,
|
|
base_url='https://openrouter.ai/api/v1'
|
|
)
|
|
from urllib.parse import urlparse
|
|
domain = urlparse(url).netloc
|
|
|
|
response = client.chat.completions.create(
|
|
model='openai/gpt-4o-mini',
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': (
|
|
f'I am a travel industry analyst conducting competitive research. '
|
|
f'Please search for this tour page and extract the following factual data points: '
|
|
f'trip name, trip code, price, duration in days, maximum group size, '
|
|
f'total included meals, hotel names, departure dates, start location, '
|
|
f'end location, service level, and key activities. '
|
|
f'Return all factual information you find.\n\nURL: {url}'
|
|
)
|
|
}],
|
|
tools=[{
|
|
'type': 'openrouter:web_search',
|
|
'parameters': {
|
|
'engine': 'exa',
|
|
'max_results': 5,
|
|
'search_context_size': 'high',
|
|
'allowed_domains': [domain]
|
|
}
|
|
}],
|
|
temperature=0.0,
|
|
max_tokens=8192
|
|
)
|
|
|
|
msg = response.choices[0].message
|
|
text = msg.content if isinstance(msg.content, str) else ''
|
|
|
|
# Also extract content from tool result annotations if present
|
|
if hasattr(msg, 'annotations') and msg.annotations:
|
|
for annotation in msg.annotations:
|
|
if hasattr(annotation, 'url_citation') and annotation.url_citation:
|
|
citation_content = getattr(annotation.url_citation, 'content', '')
|
|
if citation_content:
|
|
text = text + '\n\n' + citation_content
|
|
|
|
text = text.strip()
|
|
char_count = len(text)
|
|
logger.info(f'AI web fetch returned {char_count} chars for {url}')
|
|
logger.info(f'AI web fetch content preview: {text[:500]}')
|
|
|
|
if char_count > 300:
|
|
return {
|
|
'text': text[:25000],
|
|
'tables': '',
|
|
'url': url,
|
|
'char_count': char_count,
|
|
'success': True
|
|
}
|
|
else:
|
|
logger.warning(f'AI web fetch also returned low content for {url}')
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f'AI web fetch failed for {url}: {str(e)}')
|
|
return None
|
|
|
|
|
|
# ── PROMPT BUILDER ───────────────────────────────────────────
|
|
|
|
def build_prompt(competitor_name, page_data, product_name,
|
|
destination, duration, travel_style, is_ngs=False):
|
|
|
|
ngs_fields = '''
|
|
"exclusiveAccess": "exclusive access or special unique inclusions",
|
|
"groupLeader": "group leader and/or local expert description",
|
|
"sustainability": "sustainability inclusions or null",''' if is_ngs else ''
|
|
|
|
content = page_data['text'] if page_data else '[PAGE CONTENT WILL BE INSERTED HERE]'
|
|
if page_data and page_data.get('tables'):
|
|
content += f'\n\nTABLE DATA:\n{page_data["tables"]}'
|
|
|
|
url = page_data['url'] if page_data else '[COMPETITOR URL]'
|
|
|
|
return f'''You are a travel industry analyst helping G Adventures research competitors.
|
|
Be concise — use short phrases not full sentences for all fields except comments.
|
|
Read the ENTIRE page content carefully before extracting any fields.
|
|
Pay particular attention to itinerary sections, departure tables, and pricing grids
|
|
which may appear later in the content.
|
|
|
|
The content below was scraped from the {competitor_name} tour page at: {url}
|
|
|
|
This is the specific page selected as most comparable to:
|
|
- G Adventures Product: {product_name}
|
|
- Destination: {destination}
|
|
- Duration: ~{duration} days
|
|
- G Adventures Travel Style: {travel_style}
|
|
|
|
PAGE CONTENT:
|
|
---
|
|
{content}
|
|
---
|
|
|
|
CRITICAL PRICING RULES:
|
|
- majorityPrice: Standard/regular adult price for the most common departure.
|
|
IGNORE sale, promotional, early bird, or "was/now" pricing. Use non-discounted rack rate only.
|
|
- Prices may appear as "from $X", "per person from $X", or inside a departures table or pricing grid.
|
|
Extract the most commonly listed standard adult price regardless of how it is labelled on the page.
|
|
- priceLow: Lowest standard price across all departures shown (not sale price).
|
|
- priceHigh: Highest standard price across all departures shown (not sale price).
|
|
- If a site requires clicking "see more departures" to show all dates, note this in seasonality
|
|
and use only prices visible on the main listing page.
|
|
- Multi-currency: only fill if the price is explicitly listed in that currency on the page.
|
|
Do not convert. If the page shows GBP only, fill gbpPrice and leave USD fields null.
|
|
|
|
MEALS RULE:
|
|
- meals: Return the TOTAL number of included meals as a single integer only. No text description.
|
|
- Meals may be listed individually per day in the itinerary section
|
|
(e.g. "Day 1: Breakfast", "Day 3: Breakfast and Dinner").
|
|
Count every individual meal mention across all itinerary days.
|
|
Do not rely on a summary box — read through the full itinerary to count.
|
|
- Count each meal type separately: 1 breakfast + 1 lunch + 1 dinner = 3 meals.
|
|
|
|
Return ONLY a valid JSON object. No markdown, no code fences, no explanation.
|
|
|
|
{{
|
|
"company": "{competitor_name}",
|
|
"link": "{url}",
|
|
"tripName": "exact trip name from the page",
|
|
"tripCode": "trip code if shown, otherwise null",
|
|
"serviceLevel": "accommodation service level",
|
|
"groupSize": max group size as integer,
|
|
"duration": number of days as integer,
|
|
"meals": total included meals as integer,{ngs_fields}
|
|
"startLocation": "departure city",
|
|
"endLocation": "end city",
|
|
"departures": approximate departures per year as integer or null,
|
|
"seasonality": "price bands by season. Note if more departures are hidden behind a button.",
|
|
"startDays": "departure days of week",
|
|
"targetAudience": "target traveller",
|
|
"hotels": "hotel types or properties",
|
|
"activities": "key included activities",
|
|
"relevancy": relevancy score 1-5 as integer,
|
|
"majorityPrice": standard adult USD price as number or null,
|
|
"priceLow": lowest standard USD price as number or null,
|
|
"priceHigh": highest standard USD price as number or null,
|
|
"comments": "analysis: key differences from G Adventures, strengths, weaknesses, pricing position",
|
|
"dateUsed": "specific departure date used for majority price",
|
|
"audPrice": standard AUD price as number or null,
|
|
"cadPrice": standard CAD price as number or null,
|
|
"eurPrice": standard EUR price as number or null,
|
|
"gbpPrice": standard GBP price as number or null
|
|
}}'''
|
|
|
|
|
|
def inject_page_content(prompt_template, competitor_name, page_data):
|
|
"""Inject real scraped content into a user-edited prompt template."""
|
|
content = page_data['text']
|
|
if page_data.get('tables'):
|
|
content += f'\n\nTABLE DATA:\n{page_data["tables"]}'
|
|
result = prompt_template.replace('[PAGE CONTENT WILL BE INSERTED HERE]', content)
|
|
result = result.replace('[COMPETITOR URL]', page_data['url'])
|
|
result = result.replace(
|
|
competitor_name + ' tour page at: [COMPETITOR URL]',
|
|
competitor_name + f' tour page at: {page_data["url"]}'
|
|
)
|
|
return result
|
|
|
|
|
|
# ── EXTRACTION — OPENROUTER WITH MODEL FALLBACK ───────────────
|
|
# Active provider. Falls through model list if free tier is exhausted.
|
|
# ── TO SWITCH TO GROQ ────────────────────────────────────────
|
|
# 1. Comment out extract_with_groq below
|
|
# 2. Uncomment the Groq version further below
|
|
# ── TO SWITCH TO GEMINI ──────────────────────────────────────
|
|
# 1. Uncomment gemini import and GEMINI_API_KEY in config
|
|
# 2. Comment out extract_with_groq below
|
|
# 3. Uncomment extract_with_gemini further below
|
|
# 4. In requirements.txt swap openai for google-generativeai
|
|
# ─────────────────────────────────────────────────────────────
|
|
|
|
def extract_with_groq(competitor_name, prompt):
|
|
client = OpenAI(
|
|
api_key=OPENROUTER_API_KEY,
|
|
base_url='https://openrouter.ai/api/v1'
|
|
)
|
|
|
|
models = [
|
|
'openai/gpt-oss-120b:free',
|
|
'minimax/minimax-m2.5:free',
|
|
'nvidia/nemotron-3-super-120b-a12b:free',
|
|
'meta-llama/llama-3.3-70b-instruct:free'
|
|
]
|
|
|
|
last_model_used = None
|
|
|
|
for model in models:
|
|
for attempt in range(2):
|
|
try:
|
|
last_model_used = model
|
|
response = client.chat.completions.create(
|
|
model=model,
|
|
messages=[{'role': 'user', 'content': prompt}],
|
|
temperature=0.1,
|
|
max_tokens=4096
|
|
)
|
|
text = response.choices[0].message.content
|
|
cleaned = re.sub(r'```json|```', '', text).strip()
|
|
|
|
try:
|
|
return json.loads(cleaned)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
match = re.search(r'\{[\s\S]*\}', cleaned)
|
|
if match:
|
|
try:
|
|
return json.loads(match.group(0))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
try:
|
|
fixed = re.sub(r',\s*}', '}', cleaned)
|
|
fixed = re.sub(r',\s*]', ']', fixed)
|
|
fixed = re.sub(r'([{,])\s*([a-zA-Z0-9_]+)\s*:', r'\1"\2":', fixed)
|
|
return json.loads(fixed)
|
|
except json.JSONDecodeError:
|
|
raise ValueError(
|
|
f'Could not parse response from {model} for {competitor_name}'
|
|
)
|
|
|
|
except Exception as e:
|
|
error_str = str(e)
|
|
if '429' in error_str or 'rate_limit' in error_str.lower():
|
|
wait = 5 * (attempt + 1)
|
|
logger.warning(
|
|
f'{model} rate limited — waiting {wait}s (attempt {attempt+1}/2)...'
|
|
)
|
|
time.sleep(wait)
|
|
elif '403' in error_str and 'credit' in error_str.lower():
|
|
logger.warning(f'{model} free tier exhausted — trying next model...')
|
|
break
|
|
else:
|
|
logger.warning(f'{model} failed: {error_str[:200]}')
|
|
if attempt == 1:
|
|
logger.warning(f'Moving to next model after {model} failed twice')
|
|
|
|
raise Exception(
|
|
f'All models exhausted for {competitor_name}. Last tried: {last_model_used}'
|
|
)
|
|
|
|
|
|
# ── GROQ VERSION (commented out — uncomment to switch) ───────
|
|
#
|
|
# def extract_with_groq(competitor_name, prompt):
|
|
# client = OpenAI(
|
|
# api_key=GROQ_API_KEY,
|
|
# base_url='https://api.groq.com/openai/v1'
|
|
# )
|
|
# for attempt in range(3):
|
|
# try:
|
|
# response = client.chat.completions.create(
|
|
# model='llama-3.3-70b-versatile',
|
|
# messages=[{'role': 'user', 'content': prompt}],
|
|
# temperature=0.1,
|
|
# max_tokens=4096
|
|
# )
|
|
# text = response.choices[0].message.content
|
|
# cleaned = re.sub(r'```json|```', '', text).strip()
|
|
# try:
|
|
# return json.loads(cleaned)
|
|
# except json.JSONDecodeError:
|
|
# match = re.search(r'\{[\s\S]*\}', cleaned)
|
|
# if match:
|
|
# return json.loads(match.group(0))
|
|
# raise ValueError(f'Could not parse Groq response for {competitor_name}')
|
|
# except Exception as e:
|
|
# if '429' in str(e) and attempt < 2:
|
|
# wait = 10 * (attempt + 1)
|
|
# logger.warning(f'Rate limited — waiting {wait}s...')
|
|
# time.sleep(wait)
|
|
# else:
|
|
# raise
|
|
|
|
|
|
# ── GEMINI VERSION (commented out — uncomment to switch) ─────
|
|
#
|
|
# def extract_with_gemini(competitor_name, prompt):
|
|
# genai.configure(api_key=GEMINI_API_KEY)
|
|
# model = genai.GenerativeModel('gemini-2.5-flash')
|
|
# for attempt in range(3):
|
|
# try:
|
|
# response = model.generate_content(
|
|
# prompt,
|
|
# generation_config=genai.types.GenerationConfig(
|
|
# temperature=0.1, max_output_tokens=4096
|
|
# )
|
|
# )
|
|
# text = response.text
|
|
# cleaned = re.sub(r'```json|```', '', text).strip()
|
|
# try:
|
|
# return json.loads(cleaned)
|
|
# except json.JSONDecodeError:
|
|
# match = re.search(r'\{[\s\S]*\}', cleaned)
|
|
# if match:
|
|
# return json.loads(match.group(0))
|
|
# raise ValueError(f'Could not parse Gemini response for {competitor_name}')
|
|
# except Exception as e:
|
|
# if '429' in str(e) and attempt < 2:
|
|
# wait = 10 * (attempt + 1)
|
|
# logger.warning(f'Rate limited — waiting {wait}s...')
|
|
# time.sleep(wait)
|
|
# else:
|
|
# raise
|
|
|
|
|
|
# ── ROUTES ───────────────────────────────────────────────────
|
|
|
|
@app.route('/health', methods=['GET'])
|
|
def health():
|
|
return jsonify({'status': 'ok'})
|
|
|
|
|
|
@app.route('/competitors', methods=['GET'])
|
|
def get_competitors():
|
|
if not check_auth(request):
|
|
return jsonify({'error': 'Unauthorized'}), 401
|
|
try:
|
|
competitors = get_competitors_from_db()
|
|
return jsonify({'competitors': competitors})
|
|
except Exception as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@app.route('/preview-prompt', methods=['POST'])
|
|
def preview_prompt():
|
|
if not check_auth(request):
|
|
return jsonify({'error': 'Unauthorized'}), 401
|
|
|
|
data = request.json or {}
|
|
travel_style = data.get('travelStyle', '')
|
|
product_name = data.get('productName', '')
|
|
destination = data.get('destination', '')
|
|
duration = data.get('duration', '')
|
|
tab_type = data.get('tabType', 'Standard')
|
|
is_ngs = tab_type == 'NGS'
|
|
|
|
placeholder_page = {
|
|
'text': '[PAGE CONTENT WILL BE INSERTED HERE]',
|
|
'tables': '',
|
|
'url': '[COMPETITOR URL]'
|
|
}
|
|
prompt = build_prompt(
|
|
'[Competitor Name]', placeholder_page,
|
|
product_name, destination, duration, travel_style, is_ngs
|
|
)
|
|
return jsonify({'prompt': prompt})
|
|
|
|
|
|
@app.route('/research', methods=['POST'])
|
|
def research():
|
|
if not check_auth(request):
|
|
return jsonify({'error': 'Unauthorized'}), 401
|
|
|
|
data = request.json
|
|
if not data:
|
|
return jsonify({'error': 'No JSON body'}), 400
|
|
|
|
required = ['travelStyle', 'productName', 'destination',
|
|
'duration', 'tabType', 'competitors']
|
|
for field in required:
|
|
if field not in data:
|
|
return jsonify({'error': f'Missing field: {field}'}), 400
|
|
|
|
travel_style = data['travelStyle']
|
|
product_name = data['productName']
|
|
destination = data['destination']
|
|
duration = data['duration']
|
|
tab_type = data['tabType']
|
|
competitors = data['competitors']
|
|
prompt_template = data.get('promptTemplate', None)
|
|
|
|
is_ngs = tab_type == 'NGS'
|
|
tab_name = NGS_TAB if is_ngs else STANDARD_TAB
|
|
fields = NGS_FIELDS if is_ngs else STANDARD_FIELDS
|
|
START_COL = 4
|
|
|
|
results = {'success': [], 'failed': []}
|
|
|
|
for i, competitor in enumerate(competitors):
|
|
name = competitor['name']
|
|
url = competitor['url']
|
|
col = competitor.get('col', START_COL + i)
|
|
|
|
logger.info(f'Processing {name} — {url}')
|
|
|
|
try:
|
|
# Step 1: Scrape
|
|
page_data = scrape(url)
|
|
if not page_data['success']:
|
|
raise ValueError(page_data['text'])
|
|
logger.info(f'{name} — scraped {page_data["char_count"]} chars')
|
|
|
|
# Step 2: Build prompt
|
|
if prompt_template and len(prompt_template.strip()) > 100:
|
|
prompt = inject_page_content(prompt_template, name, page_data)
|
|
else:
|
|
prompt = build_prompt(
|
|
name, page_data, product_name,
|
|
destination, duration, travel_style, is_ngs
|
|
)
|
|
|
|
# Step 3: Extract
|
|
extracted = extract_with_groq(name, prompt)
|
|
logger.info(f'{name} — extracted: {extracted.get("tripName")}')
|
|
|
|
# Step 4: Write to sheet
|
|
write_to_sheet(tab_name, col, extracted, fields)
|
|
logger.info(f'{name} — written to col {col}')
|
|
|
|
results['success'].append({
|
|
'name': name,
|
|
'tripName': extracted.get('tripName'),
|
|
'price': extracted.get('majorityPrice'),
|
|
'col': col
|
|
})
|
|
|
|
except Exception as e:
|
|
logger.error(f'{name} failed: {str(e)}')
|
|
results['failed'].append({'name': name, 'error': str(e)})
|
|
try:
|
|
client = get_sheet_client()
|
|
sheet = client.open_by_key(SHEET_ID)
|
|
ws = sheet.worksheet(tab_name)
|
|
ws.update_cell(fields['company'], col, f'⚠️ {name} — failed')
|
|
except Exception:
|
|
pass
|
|
|
|
return jsonify({
|
|
'successCount': len(results['success']),
|
|
'failedCount': len(results['failed']),
|
|
'success': results['success'],
|
|
'failed': results['failed']
|
|
})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
app.run(host='0.0.0.0', port=5000, debug=False) |