app.bettersight.io/backend/services/trip_finder_service.py

import os
import logging
import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
from services.url_utils import is_generic_homepage

logger = logging.getLogger(__name__)

FIRECRAWL_URL = os.getenv('FIRECRAWL_URL', 'http://firecrawl:3002')

CATEGORY_KEYWORDS = ['classic', 'trek', 'trail', 'safari', 'adventure', 'explorer', 'journey']
CONFIDENCE_FOUND_THRESHOLD = 0.3
MAX_CONCURRENT_MAP_CALLS = 5


def score_and_rank_urls(urls, destination, duration):
    """
    Scores candidate URLs by keyword match against destination and
    duration, returning the highest-scoring URL with a confidence score.

    A URL must mention at least one destination keyword to be considered
    at all — this is a hard gate, not an additive bonus. Regression found
    in production: a URL for a completely different country could still
    cross CONFIDENCE_FOUND_THRESHOLD purely from a duration-digit match
    plus a generic category-keyword match (e.g. a Turkey trip page scored
    0.75 confidence against a Peru search just because it happened to be a
    "9 day classic" tour) — destination relevance must be a prerequisite,
    not something duration/category signals alone can compensate for.

    A bare root-domain URL (no path at all) is excluded the same way, for
    the same reason: Firecrawl's /map candidate list commonly includes the
    site's own root URL, and if the destination keyword happens to appear
    in the domain itself (e.g. a competitor literally named
    "peru-travel.com"), the root URL could otherwise score high enough to
    be returned as a confident "found" match — silently presenting a
    homepage as if it were a trip-specific page, the exact failure mode
    this whole scoring function exists to prevent.

    Data flow:
      urls + destination + duration →
      per-url: zero destination-keyword matches, or a bare root domain →
      excluded entirely →
      surviving urls scored: +1 per destination keyword in URL, +2 if
      duration appears in URL, +1 per travel-category keyword in URL →
      sort by score → confidence = top_score / max_possible_score →
      { url, confidence, found } — found=False (prompt manual entry)
      when no url survives the gate, or confidence is below
      CONFIDENCE_FOUND_THRESHOLD
    """
    dest_keywords = destination.lower().split()
    scores = []

    for url in urls:
        if is_generic_homepage(url):
            continue  # bare root domain — never a trip-specific candidate

        url_lower = url.lower()
        dest_matches = sum(1 for kw in dest_keywords if kw in url_lower)
        if dest_matches == 0:
            continue  # zero destination relevance — never a candidate, regardless of duration/category signal

        score = dest_matches
        if str(duration) in url_lower:
            score += 2
        score += sum(1 for kw in CATEGORY_KEYWORDS if kw in url_lower)
        scores.append((score, url))

    scores.sort(key=lambda pair: pair[0], reverse=True)
    if not scores:
        return {'url': None, 'confidence': 0, 'found': False}

    top_score, top_url = scores[0]
    max_possible = len(dest_keywords) + 2 + 1
    confidence = min(top_score / max_possible, 1.0) if max_possible else 0

    return {
        'url': top_url,
        'confidence': round(confidence, 2),
        'found': confidence >= CONFIDENCE_FOUND_THRESHOLD,
    }


def find_competitor_trip_url(root_url, destination, duration, travel_style):
    """
    Uses Firecrawl /map with search terms to find the most likely trip
    page on a competitor site matching the PM's research intent.

    Data flow:
      root_url + destination/duration/travel_style →
      POST {FIRECRAWL_URL}/v1/map with a search string →
      candidate URL list → score_and_rank_urls() →
      { url, confidence, found } returned (found=False, url=None on any
      Firecrawl failure — never raises, since one competitor's failure
      must not block the others per CLAUDE.md §18 resilience rules)
    """
    try:
        response = requests.post(
            f'{FIRECRAWL_URL}/v1/map',
            json={'url': root_url, 'search': f'{destination} {duration} days {travel_style} tour'},
            timeout=15
        )
        response.raise_for_status()
        urls = response.json().get('links', [])
    except requests.RequestException as e:
        logger.error(f'Firecrawl /map failed for {root_url}: {str(e)}')
        return {'url': None, 'confidence': 0, 'found': False}

    if not urls:
        return {'url': None, 'confidence': 0, 'found': False}

    return score_and_rank_urls(urls, destination, duration)


def find_all_competitor_urls(competitors, destination, duration, travel_style):
    """
    Runs trip URL finding for all selected competitors concurrently —
    at most MAX_CONCURRENT_MAP_CALLS Firecrawl /map calls in flight at
    once, respecting server limits.

    Data flow:
      competitors list → ThreadPoolExecutor(max_workers=5) →
      find_competitor_trip_url() per competitor in parallel →
      collect results as they complete → matches list returned
      (one entry per competitor, in completion order)
    """
    results = []
    with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_MAP_CALLS) as executor:
        futures = {
            executor.submit(
                find_competitor_trip_url, c['website'], destination, duration, travel_style
            ): c
            for c in competitors
        }
        for future in as_completed(futures):
            competitor = futures[future]
            match = future.result()
            results.append({
                'competitor_id': competitor['id'],
                'competitor_name': competitor['name'],
                **match,
            })
    return results