import os
import json
import re
import asyncio
import time
import logging
from flask import Flask, request, jsonify
from dotenv import load_dotenv
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
# import google.generativeai as genai  # ── Uncomment to use Gemini instead of OpenRouter
import gspread
from openai import OpenAI
from google.oauth2.service_account import Credentials

load_dotenv()

app = Flask(__name__)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# ── CONFIG ───────────────────────────────────────────────────

# GEMINI_API_KEY   = os.getenv('GEMINI_API_KEY')  # ── Uncomment to use Gemini
GROQ_API_KEY         = os.getenv('GROQ_API_KEY')
OPENROUTER_API_KEY   = os.getenv('OPENROUTER_API_KEY')
API_SECRET_KEY       = os.getenv('API_SECRET_KEY')
SHEET_ID             = os.getenv('SHEET_ID')
STANDARD_TAB         = os.getenv('STANDARD_TAB', 'Standard Competitive Tab')
NGS_TAB              = os.getenv('NGS_TAB', 'NGS Competitive Tab')
SERVICE_ACCOUNT_JSON = os.getenv('GOOGLE_SERVICE_ACCOUNT_JSON')

# ── FIELD MAPS ───────────────────────────────────────────────

STANDARD_FIELDS = {
    'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
    'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
    'startLocation': 15, 'endLocation': 16, 'departures': 17,
    'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
    'hotels': 22, 'activities': 23, 'relevancy': 24,
    'majorityPrice': 25, 'priceLow': 26, 'priceHigh': 27,
    'comments': 30, 'dateUsed': 31,
    'audPrice': 32, 'cadPrice': 33, 'eurPrice': 34, 'gbpPrice': 35
}

NGS_FIELDS = {
    'company': 6, 'link': 8, 'tripName': 9, 'tripCode': 10,
    'serviceLevel': 11, 'groupSize': 12, 'duration': 13, 'meals': 14,
    'startLocation': 15, 'endLocation': 16, 'departures': 17,
    'seasonality': 19, 'startDays': 20, 'targetAudience': 21,
    'hotels': 22, 'activities': 23, 'exclusiveAccess': 24,
    'groupLeader': 25, 'sustainability': 26, 'relevancy': 27,
    'majorityPrice': 28, 'priceLow': 29, 'priceHigh': 30,
    'comments': 33, 'dateUsed': 34,
    'audPrice': 35, 'cadPrice': 36, 'eurPrice': 37, 'gbpPrice': 38
}


# ── AUTH ─────────────────────────────────────────────────────

def check_auth(req):
    token = req.headers.get('X-API-Key')
    if not token or token != API_SECRET_KEY:
        return False
    return True


# ── GOOGLE SHEETS ────────────────────────────────────────────

def get_sheet_client():
    scopes = [
        'https://www.googleapis.com/auth/spreadsheets',
        'https://www.googleapis.com/auth/drive'
    ]
    info  = json.loads(SERVICE_ACCOUNT_JSON)
    creds = Credentials.from_service_account_info(info, scopes=scopes)
    return gspread.authorize(creds)


def write_to_sheet(tab_name, col_index, data, fields):
    client = get_sheet_client()
    sheet  = client.open_by_key(SHEET_ID)
    ws     = sheet.worksheet(tab_name)
    updates = []
    for field, row in fields.items():
        value = data.get(field)
        if value is None or value == '':
            continue
        updates.append({
            'range':  gspread.utils.rowcol_to_a1(row, col_index),
            'values': [[value]]
        })
    if updates:
        ws.batch_update(updates)


def get_competitors_from_db():
    client = get_sheet_client()
    sheet  = client.open_by_key(SHEET_ID)
    ws     = sheet.worksheet('Competitor DB')
    rows   = ws.get_all_records()
    return [
        {
            'style':  r['Travel Style'],
            'name':   r['Competitor Name'],
            'url':    r['Website URL'],
            'active': r['Active']
        }
        for r in rows
        if str(r.get('Active', '')).strip().lower() == 'yes'
    ]


# ── PLAYWRIGHT SCRAPER ───────────────────────────────────────

async def render_page(url: str) -> dict:
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            args=['--no-sandbox', '--disable-setuid-sandbox',
                  '--disable-dev-shm-usage', '--disable-gpu']
        )
        context = await browser.new_context(
            user_agent=(
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                'AppleWebKit/537.36 (KHTML, like Gecko) '
                'Chrome/124.0.0.0 Safari/537.36'
            ),
            viewport={'width': 1366, 'height': 768},
            locale='en-GB',
            timezone_id='Europe/London',
            extra_http_headers={
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
                'Accept-Language': 'en-GB,en;q=0.9',
                'Accept-Encoding': 'gzip, deflate, br',
                'DNT': '1',
                'Upgrade-Insecure-Requests': '1',
                'Sec-Fetch-Dest': 'document',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'none',
                'Sec-Fetch-User': '?1',
            }
        )
        page = await context.new_page()

        # Basic stealth — manual navigator spoofing
        await page.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
            Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
            Object.defineProperty(navigator, 'languages', { get: () => ['en-GB', 'en'] });
            window.chrome = { runtime: {} };
        """)

        # Block only known third-party analytics/ad domains
        BLOCKED_DOMAINS = [
            'google-analytics.com', 'googletagmanager.com', 'doubleclick.net',
            'facebook.net', 'facebook.com/tr', 'hotjar.com', 'intercom.io',
            'zendesk.com', 'hubspot.com', 'marketo.com', 'segment.com',
            'amplitude.com', 'mixpanel.com', 'optimizely.com', 'crazyegg.com',
        ]

        async def block_trackers(route):
            if any(domain in route.request.url for domain in BLOCKED_DOMAINS):
                await route.abort()
            else:
                await route.continue_()

        await page.route('**/*', block_trackers)

        try:
            try:
                await page.goto(url, wait_until='domcontentloaded', timeout=30000)
                logger.info(f'Loaded: {url}')
            except Exception as e:
                logger.warning(f'domcontentloaded failed, trying networkidle: {str(e)[:100]}')
                await page.goto(url, wait_until='networkidle', timeout=60000)

            # Dismiss cookie consent banners
            try:
                await page.click(
                    'button:has-text("Accept All"), button:has-text("Accept all"), '
                    'button:has-text("Accept"), button:has-text("OK"), '
                    'button:has-text("I agree"), button:has-text("Agree"), '
                    '[id*="cookie"] button, [class*="cookie"] button, '
                    '[id*="consent"] button, [class*="consent"] button',
                    timeout=3000
                )
                logger.info('Cookie banner dismissed')
                await page.wait_for_timeout(1000)
            except Exception:
                pass

            # Wait for meaningful content
            try:
                await page.wait_for_selector(
                    'h1, .trip-overview, main, article, #overview, '
                    '[class*="itinerary"], [class*="price"], [class*="trip"]',
                    timeout=8000
                )
            except Exception:
                pass

            await page.wait_for_timeout(3000)

            # Two scroll passes to trigger lazy-loaded content
            for _ in range(2):
                await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
                await page.wait_for_timeout(1500)
                await page.evaluate('window.scrollTo(0, 0)')
                await page.wait_for_timeout(500)

            await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
            await page.wait_for_timeout(2000)

            html = await page.content()
            soup = BeautifulSoup(html, 'lxml')

            for tag in soup(['script', 'style', 'nav', 'footer',
                             'header', 'iframe', 'noscript', 'svg',
                             'button', 'form']):
                tag.decompose()

            # Prioritise main content area
            main_content = (
                soup.find('main') or
                soup.find('article') or
                soup.find(id='main') or
                soup.find(id='content') or
                soup.find(class_='content') or
                soup
            )

            text = main_content.get_text(separator=' ', strip=True)
            text = re.sub(r'\s+', ' ', text).strip()

            tables = [t.get_text(separator=' | ', strip=True)
                      for t in soup.find_all('table')]
            table_text = '\n'.join(tables)
            char_count = len(text)

            if char_count < 500:
                logger.warning(
                    f'Low char count ({char_count}) for {url} — '
                    f'will attempt AI web fetch fallback'
                )

            return {
                'text':       text[:25000],
                'tables':     table_text[:5000],
                'url':        url,
                'char_count': char_count,
                'success':    True
            }

        except Exception as e:
            logger.error(f'Render failed for {url}: {str(e)}')
            return {
                'text': f'Render failed: {str(e)}',
                'tables': '', 'url': url,
                'char_count': 0, 'success': False
            }
        finally:
            await browser.close()


def clean_url(url: str) -> str:
    """Strip URL fragments and common tracking parameters."""
    from urllib.parse import urlparse, urlencode, parse_qs, urlunparse
    parsed = urlparse(url)
    STRIP_PARAMS = {
        'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
        'referrerView', 'ref', 'referrer', 'source', 'fbclid', 'gclid',
        'mc_cid', 'mc_eid', '_ga', 'affiliate'
    }
    if parsed.query:
        params = parse_qs(parsed.query, keep_blank_values=True)
        cleaned = {k: v for k, v in params.items() if k not in STRIP_PARAMS}
        query = urlencode(cleaned, doseq=True)
    else:
        query = ''
    clean = urlunparse((
        parsed.scheme, parsed.netloc, parsed.path,
        parsed.params, query, ''
    ))
    if clean != url:
        logger.info(f'URL cleaned: {url} → {clean}')
    return clean


def scrape(url):
    import nest_asyncio
    nest_asyncio.apply()
    url = clean_url(url)
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    try:
        result = loop.run_until_complete(render_page(url))
    finally:
        loop.close()

    # AI web fetch fallback for bot-protected sites
    if result.get('char_count', 0) < 1000 and result.get('success'):
        logger.info(f'Playwright char count low — trying AI web fetch for {url}')
        ai_result = fetch_with_ai(url)
        if ai_result:
            return ai_result

    return result


# ── AI WEB FETCH FALLBACK ────────────────────────────────────
# Fires only when Playwright returns < 500 chars.
# Uses OpenRouter web_search server tool with Exa engine.
# Cost: ~$0.02 per call + token cost. Only fires for bot-protected sites.

def fetch_with_ai(url: str) -> dict:
    logger.info(f'Playwright failed — attempting AI web fetch for {url}')
    try:
        client = OpenAI(
            api_key=OPENROUTER_API_KEY,
            base_url='https://openrouter.ai/api/v1'
        )
        from urllib.parse import urlparse
        domain = urlparse(url).netloc

        response = client.chat.completions.create(
            model='openai/gpt-4o-mini',
            messages=[{
                'role': 'user',
                'content': (
                    f'I am a travel industry analyst conducting competitive research. '
                    f'Please search for this tour page and extract the following factual data points: '
                    f'trip name, trip code, price, duration in days, maximum group size, '
                    f'total included meals, hotel names, departure dates, start location, '
                    f'end location, service level, and key activities. '
                    f'Return all factual information you find.\n\nURL: {url}'
                )
            }],
            tools=[{
                'type': 'openrouter:web_search',
                'parameters': {
                    'engine': 'exa',
                    'max_results': 5,
                    'search_context_size': 'high',
                    'allowed_domains': [domain]
                }
            }],
            temperature=0.0,
            max_tokens=8192
        )

        msg = response.choices[0].message
        text = msg.content if isinstance(msg.content, str) else ''

        # Also extract content from tool result annotations if present
        if hasattr(msg, 'annotations') and msg.annotations:
            for annotation in msg.annotations:
                if hasattr(annotation, 'url_citation') and annotation.url_citation:
                    citation_content = getattr(annotation.url_citation, 'content', '')
                    if citation_content:
                        text = text + '\n\n' + citation_content

        text = text.strip()
        char_count = len(text)
        logger.info(f'AI web fetch returned {char_count} chars for {url}')
        logger.info(f'AI web fetch content preview: {text[:500]}')

        if char_count > 300:
            return {
                'text':       text[:25000],
                'tables':     '',
                'url':        url,
                'char_count': char_count,
                'success':    True
            }
        else:
            logger.warning(f'AI web fetch also returned low content for {url}')
            return None

    except Exception as e:
        logger.error(f'AI web fetch failed for {url}: {str(e)}')
        return None


# ── PROMPT BUILDER ───────────────────────────────────────────

def build_prompt(competitor_name, page_data, product_name,
                 destination, duration, travel_style, is_ngs=False):

    ngs_fields = '''
  "exclusiveAccess": "exclusive access or special unique inclusions",
  "groupLeader": "group leader and/or local expert description",
  "sustainability": "sustainability inclusions or null",''' if is_ngs else ''

    content = page_data['text'] if page_data else '[PAGE CONTENT WILL BE INSERTED HERE]'
    if page_data and page_data.get('tables'):
        content += f'\n\nTABLE DATA:\n{page_data["tables"]}'

    url = page_data['url'] if page_data else '[COMPETITOR URL]'

    return f'''You are a travel industry analyst helping G Adventures research competitors.
Be concise — use short phrases not full sentences for all fields except comments.
Read the ENTIRE page content carefully before extracting any fields.
Pay particular attention to itinerary sections, departure tables, and pricing grids
which may appear later in the content.

The content below was scraped from the {competitor_name} tour page at: {url}

This is the specific page selected as most comparable to:
- G Adventures Product: {product_name}
- Destination: {destination}
- Duration: ~{duration} days
- G Adventures Travel Style: {travel_style}

PAGE CONTENT:
---
{content}
---

CRITICAL PRICING RULES:
- majorityPrice: Standard/regular adult price for the most common departure.
  IGNORE sale, promotional, early bird, or "was/now" pricing. Use non-discounted rack rate only.
- Prices may appear as "from $X", "per person from $X", or inside a departures table or pricing grid.
  Extract the most commonly listed standard adult price regardless of how it is labelled on the page.
- priceLow: Lowest standard price across all departures shown (not sale price).
- priceHigh: Highest standard price across all departures shown (not sale price).
- If a site requires clicking "see more departures" to show all dates, note this in seasonality
  and use only prices visible on the main listing page.
- Multi-currency: only fill if the price is explicitly listed in that currency on the page.
  Do not convert. If the page shows GBP only, fill gbpPrice and leave USD fields null.

MEALS RULE:
- meals: Return the TOTAL number of included meals as a single integer only. No text description.
- Meals may be listed individually per day in the itinerary section
  (e.g. "Day 1: Breakfast", "Day 3: Breakfast and Dinner").
  Count every individual meal mention across all itinerary days.
  Do not rely on a summary box — read through the full itinerary to count.
- Count each meal type separately: 1 breakfast + 1 lunch + 1 dinner = 3 meals.

Return ONLY a valid JSON object. No markdown, no code fences, no explanation.

{{
  "company": "{competitor_name}",
  "link": "{url}",
  "tripName": "exact trip name from the page",
  "tripCode": "trip code if shown, otherwise null",
  "serviceLevel": "accommodation service level",
  "groupSize": max group size as integer,
  "duration": number of days as integer,
  "meals": total included meals as integer,{ngs_fields}
  "startLocation": "departure city",
  "endLocation": "end city",
  "departures": approximate departures per year as integer or null,
  "seasonality": "price bands by season. Note if more departures are hidden behind a button.",
  "startDays": "departure days of week",
  "targetAudience": "target traveller",
  "hotels": "hotel types or properties",
  "activities": "key included activities",
  "relevancy": relevancy score 1-5 as integer,
  "majorityPrice": standard adult USD price as number or null,
  "priceLow": lowest standard USD price as number or null,
  "priceHigh": highest standard USD price as number or null,
  "comments": "analysis: key differences from G Adventures, strengths, weaknesses, pricing position",
  "dateUsed": "specific departure date used for majority price",
  "audPrice": standard AUD price as number or null,
  "cadPrice": standard CAD price as number or null,
  "eurPrice": standard EUR price as number or null,
  "gbpPrice": standard GBP price as number or null
}}'''


def inject_page_content(prompt_template, competitor_name, page_data):
    """Inject real scraped content into a user-edited prompt template."""
    content = page_data['text']
    if page_data.get('tables'):
        content += f'\n\nTABLE DATA:\n{page_data["tables"]}'
    result = prompt_template.replace('[PAGE CONTENT WILL BE INSERTED HERE]', content)
    result = result.replace('[COMPETITOR URL]', page_data['url'])
    result = result.replace(
        competitor_name + ' tour page at: [COMPETITOR URL]',
        competitor_name + f' tour page at: {page_data["url"]}'
    )
    return result


# ── EXTRACTION — OPENROUTER WITH MODEL FALLBACK ───────────────
# Active provider. Falls through model list if free tier is exhausted.
# ── TO SWITCH TO GROQ ────────────────────────────────────────
# 1. Comment out extract_with_groq below
# 2. Uncomment the Groq version further below
# ── TO SWITCH TO GEMINI ──────────────────────────────────────
# 1. Uncomment gemini import and GEMINI_API_KEY in config
# 2. Comment out extract_with_groq below
# 3. Uncomment extract_with_gemini further below
# 4. In requirements.txt swap openai for google-generativeai
# ─────────────────────────────────────────────────────────────

def extract_with_groq(competitor_name, prompt):
    client = OpenAI(
        api_key=OPENROUTER_API_KEY,
        base_url='https://openrouter.ai/api/v1'
    )

    models = [
        'openai/gpt-oss-120b:free',
        'minimax/minimax-m2.5:free',
        'nvidia/nemotron-3-super-120b-a12b:free',
        'meta-llama/llama-3.3-70b-instruct:free'
    ]

    last_model_used = None

    for model in models:
        for attempt in range(2):
            try:
                last_model_used = model
                response = client.chat.completions.create(
                    model=model,
                    messages=[{'role': 'user', 'content': prompt}],
                    temperature=0.1,
                    max_tokens=4096
                )
                text    = response.choices[0].message.content
                cleaned = re.sub(r'```json|```', '', text).strip()

                try:
                    return json.loads(cleaned)
                except json.JSONDecodeError:
                    pass

                match = re.search(r'\{[\s\S]*\}', cleaned)
                if match:
                    try:
                        return json.loads(match.group(0))
                    except json.JSONDecodeError:
                        pass

                try:
                    fixed = re.sub(r',\s*}', '}', cleaned)
                    fixed = re.sub(r',\s*]', ']', fixed)
                    fixed = re.sub(r'([{,])\s*([a-zA-Z0-9_]+)\s*:', r'\1"\2":', fixed)
                    return json.loads(fixed)
                except json.JSONDecodeError:
                    raise ValueError(
                        f'Could not parse response from {model} for {competitor_name}'
                    )

            except Exception as e:
                error_str = str(e)
                if '429' in error_str or 'rate_limit' in error_str.lower():
                    wait = 5 * (attempt + 1)
                    logger.warning(
                        f'{model} rate limited — waiting {wait}s (attempt {attempt+1}/2)...'
                    )
                    time.sleep(wait)
                elif '403' in error_str and 'credit' in error_str.lower():
                    logger.warning(f'{model} free tier exhausted — trying next model...')
                    break
                else:
                    logger.warning(f'{model} failed: {error_str[:200]}')
                    if attempt == 1:
                        logger.warning(f'Moving to next model after {model} failed twice')

    raise Exception(
        f'All models exhausted for {competitor_name}. Last tried: {last_model_used}'
    )


# ── GROQ VERSION (commented out — uncomment to switch) ───────
#
# def extract_with_groq(competitor_name, prompt):
#     client = OpenAI(
#         api_key=GROQ_API_KEY,
#         base_url='https://api.groq.com/openai/v1'
#     )
#     for attempt in range(3):
#         try:
#             response = client.chat.completions.create(
#                 model='llama-3.3-70b-versatile',
#                 messages=[{'role': 'user', 'content': prompt}],
#                 temperature=0.1,
#                 max_tokens=4096
#             )
#             text    = response.choices[0].message.content
#             cleaned = re.sub(r'```json|```', '', text).strip()
#             try:
#                 return json.loads(cleaned)
#             except json.JSONDecodeError:
#                 match = re.search(r'\{[\s\S]*\}', cleaned)
#                 if match:
#                     return json.loads(match.group(0))
#                 raise ValueError(f'Could not parse Groq response for {competitor_name}')
#         except Exception as e:
#             if '429' in str(e) and attempt < 2:
#                 wait = 10 * (attempt + 1)
#                 logger.warning(f'Rate limited — waiting {wait}s...')
#                 time.sleep(wait)
#             else:
#                 raise


# ── GEMINI VERSION (commented out — uncomment to switch) ─────
#
# def extract_with_gemini(competitor_name, prompt):
#     genai.configure(api_key=GEMINI_API_KEY)
#     model = genai.GenerativeModel('gemini-2.5-flash')
#     for attempt in range(3):
#         try:
#             response = model.generate_content(
#                 prompt,
#                 generation_config=genai.types.GenerationConfig(
#                     temperature=0.1, max_output_tokens=4096
#                 )
#             )
#             text    = response.text
#             cleaned = re.sub(r'```json|```', '', text).strip()
#             try:
#                 return json.loads(cleaned)
#             except json.JSONDecodeError:
#                 match = re.search(r'\{[\s\S]*\}', cleaned)
#                 if match:
#                     return json.loads(match.group(0))
#                 raise ValueError(f'Could not parse Gemini response for {competitor_name}')
#         except Exception as e:
#             if '429' in str(e) and attempt < 2:
#                 wait = 10 * (attempt + 1)
#                 logger.warning(f'Rate limited — waiting {wait}s...')
#                 time.sleep(wait)
#             else:
#                 raise


# ── ROUTES ───────────────────────────────────────────────────

@app.route('/health', methods=['GET'])
def health():
    return jsonify({'status': 'ok'})


@app.route('/competitors', methods=['GET'])
def get_competitors():
    if not check_auth(request):
        return jsonify({'error': 'Unauthorized'}), 401
    try:
        competitors = get_competitors_from_db()
        return jsonify({'competitors': competitors})
    except Exception as e:
        return jsonify({'error': str(e)}), 500


@app.route('/preview-prompt', methods=['POST'])
def preview_prompt():
    if not check_auth(request):
        return jsonify({'error': 'Unauthorized'}), 401

    data         = request.json or {}
    travel_style = data.get('travelStyle', '')
    product_name = data.get('productName', '')
    destination  = data.get('destination', '')
    duration     = data.get('duration', '')
    tab_type     = data.get('tabType', 'Standard')
    is_ngs       = tab_type == 'NGS'

    placeholder_page = {
        'text':   '[PAGE CONTENT WILL BE INSERTED HERE]',
        'tables': '',
        'url':    '[COMPETITOR URL]'
    }
    prompt = build_prompt(
        '[Competitor Name]', placeholder_page,
        product_name, destination, duration, travel_style, is_ngs
    )
    return jsonify({'prompt': prompt})


@app.route('/research', methods=['POST'])
def research():
    if not check_auth(request):
        return jsonify({'error': 'Unauthorized'}), 401

    data = request.json
    if not data:
        return jsonify({'error': 'No JSON body'}), 400

    required = ['travelStyle', 'productName', 'destination',
                'duration', 'tabType', 'competitors']
    for field in required:
        if field not in data:
            return jsonify({'error': f'Missing field: {field}'}), 400

    travel_style    = data['travelStyle']
    product_name    = data['productName']
    destination     = data['destination']
    duration        = data['duration']
    tab_type        = data['tabType']
    competitors     = data['competitors']
    prompt_template = data.get('promptTemplate', None)

    is_ngs    = tab_type == 'NGS'
    tab_name  = NGS_TAB if is_ngs else STANDARD_TAB
    fields    = NGS_FIELDS if is_ngs else STANDARD_FIELDS
    START_COL = 4

    results = {'success': [], 'failed': []}

    for i, competitor in enumerate(competitors):
        name = competitor['name']
        url  = competitor['url']
        col  = competitor.get('col', START_COL + i)

        logger.info(f'Processing {name} — {url}')

        try:
            # Step 1: Scrape
            page_data = scrape(url)
            if not page_data['success']:
                raise ValueError(page_data['text'])
            logger.info(f'{name} — scraped {page_data["char_count"]} chars')

            # Step 2: Build prompt
            if prompt_template and len(prompt_template.strip()) > 100:
                prompt = inject_page_content(prompt_template, name, page_data)
            else:
                prompt = build_prompt(
                    name, page_data, product_name,
                    destination, duration, travel_style, is_ngs
                )

            # Step 3: Extract
            extracted = extract_with_groq(name, prompt)
            logger.info(f'{name} — extracted: {extracted.get("tripName")}')

            # Step 4: Write to sheet
            write_to_sheet(tab_name, col, extracted, fields)
            logger.info(f'{name} — written to col {col}')

            results['success'].append({
                'name':     name,
                'tripName': extracted.get('tripName'),
                'price':    extracted.get('majorityPrice'),
                'col':      col
            })

        except Exception as e:
            logger.error(f'{name} failed: {str(e)}')
            results['failed'].append({'name': name, 'error': str(e)})
            try:
                client = get_sheet_client()
                sheet  = client.open_by_key(SHEET_ID)
                ws     = sheet.worksheet(tab_name)
                ws.update_cell(fields['company'], col, f'⚠️ {name} — failed')
            except Exception:
                pass

    return jsonify({
        'successCount': len(results['success']),
        'failedCount':  len(results['failed']),
        'success':      results['success'],
        'failed':       results['failed']
    })


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=False)