170 lines
7.1 KiB
Python
170 lines
7.1 KiB
Python
import os
|
|
import logging
|
|
import requests
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from services.url_utils import is_generic_homepage
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FIRECRAWL_URL = os.getenv('FIRECRAWL_URL', 'http://firecrawl:3002')
|
|
|
|
CATEGORY_KEYWORDS = ['classic', 'trek', 'trail', 'safari', 'adventure', 'explorer', 'journey']
|
|
CONFIDENCE_FOUND_THRESHOLD = 0.3
|
|
MAX_CONCURRENT_MAP_CALLS = 5
|
|
# Firecrawl /map fetches the target site's sitemap.xml before scoring
|
|
# candidates — a real production site with a large catalogue (thousands
|
|
# of URLs, e.g. a tenant's own established storefront rather than a
|
|
# smaller competitor test site) can take well over 15s just for that
|
|
# fetch. Confirmed directly: a real /map call against a large site's
|
|
# sitemap took ~21.6s, which a 15s timeout silently turned into a false
|
|
# "no match found" (requests.RequestException swallowed below) even
|
|
# though a retry with more time succeeded in ~2s. 30s covers this with
|
|
# margin while still failing within a bounded time on a truly dead site.
|
|
FIRECRAWL_MAP_TIMEOUT_SECONDS = 30
|
|
|
|
|
|
def score_and_rank_urls(urls, destination, duration):
|
|
"""
|
|
Scores candidate URLs by keyword match against destination and
|
|
duration, returning the highest-scoring URL with a confidence score.
|
|
|
|
A URL must mention at least one destination keyword to be considered
|
|
at all — this is a hard gate, not an additive bonus. Regression found
|
|
in production: a URL for a completely different country could still
|
|
cross CONFIDENCE_FOUND_THRESHOLD purely from a duration-digit match
|
|
plus a generic category-keyword match (e.g. a Turkey trip page scored
|
|
0.75 confidence against a Peru search just because it happened to be a
|
|
"9 day classic" tour) — destination relevance must be a prerequisite,
|
|
not something duration/category signals alone can compensate for.
|
|
|
|
A bare root-domain URL (no path at all) is excluded the same way, for
|
|
the same reason: Firecrawl's /map candidate list commonly includes the
|
|
site's own root URL, and if the destination keyword happens to appear
|
|
in the domain itself (e.g. a competitor literally named
|
|
"peru-travel.com"), the root URL could otherwise score high enough to
|
|
be returned as a confident "found" match — silently presenting a
|
|
homepage as if it were a trip-specific page, the exact failure mode
|
|
this whole scoring function exists to prevent.
|
|
|
|
Data flow:
|
|
urls + destination + duration →
|
|
per-url: zero destination-keyword matches, or a bare root domain →
|
|
excluded entirely →
|
|
surviving urls scored: +1 per destination keyword in URL, +2 if
|
|
duration appears in URL, +1 per travel-category keyword in URL →
|
|
sort by score → confidence = top_score / max_possible_score →
|
|
{ url, confidence, found } — found=False (prompt manual entry)
|
|
when no url survives the gate, or confidence is below
|
|
CONFIDENCE_FOUND_THRESHOLD
|
|
"""
|
|
dest_keywords = destination.lower().split()
|
|
scores = []
|
|
|
|
for url in urls:
|
|
if is_generic_homepage(url):
|
|
continue # bare root domain — never a trip-specific candidate
|
|
|
|
url_lower = url.lower()
|
|
dest_matches = sum(1 for kw in dest_keywords if kw in url_lower)
|
|
if dest_matches == 0:
|
|
continue # zero destination relevance — never a candidate, regardless of duration/category signal
|
|
|
|
score = dest_matches
|
|
if str(duration) in url_lower:
|
|
score += 2
|
|
score += sum(1 for kw in CATEGORY_KEYWORDS if kw in url_lower)
|
|
scores.append((score, url))
|
|
|
|
scores.sort(key=lambda pair: pair[0], reverse=True)
|
|
if not scores:
|
|
return {'url': None, 'confidence': 0, 'found': False}
|
|
|
|
top_score, top_url = scores[0]
|
|
max_possible = len(dest_keywords) + 2 + 1
|
|
confidence = min(top_score / max_possible, 1.0) if max_possible else 0
|
|
|
|
return {
|
|
'url': top_url,
|
|
'confidence': round(confidence, 2),
|
|
'found': confidence >= CONFIDENCE_FOUND_THRESHOLD,
|
|
}
|
|
|
|
|
|
def find_competitor_trip_url(root_url, destination, duration, travel_style):
|
|
"""
|
|
Uses Firecrawl /map with search terms to find the most likely trip
|
|
page on a competitor site matching the PM's research intent.
|
|
|
|
Data flow:
|
|
root_url + destination/duration/travel_style →
|
|
POST {FIRECRAWL_URL}/v1/map with a search string →
|
|
candidate URL list → score_and_rank_urls() →
|
|
{ url, confidence, found } returned (found=False, url=None on any
|
|
Firecrawl failure — never raises, since one competitor's failure
|
|
must not block the others per CLAUDE.md §18 resilience rules)
|
|
"""
|
|
try:
|
|
response = requests.post(
|
|
f'{FIRECRAWL_URL}/v1/map',
|
|
json={'url': root_url, 'search': f'{destination} {duration} days {travel_style} tour'},
|
|
timeout=FIRECRAWL_MAP_TIMEOUT_SECONDS
|
|
)
|
|
response.raise_for_status()
|
|
urls = response.json().get('links', [])
|
|
except requests.RequestException as e:
|
|
logger.error(f'Firecrawl /map failed for {root_url}: {str(e)}')
|
|
return {'url': None, 'confidence': 0, 'found': False}
|
|
|
|
if not urls:
|
|
return {'url': None, 'confidence': 0, 'found': False}
|
|
|
|
return score_and_rank_urls(urls, destination, duration)
|
|
|
|
|
|
def find_own_trip_url(website, destination, duration, travel_style):
|
|
"""
|
|
Finds the tenant's own best-matching trip page on their own site,
|
|
using the exact same Firecrawl /map + scoring gate as competitor
|
|
matching (score_and_rank_urls()'s destination-keyword hard gate and
|
|
root-domain exclusion apply unchanged) — named separately purely for
|
|
call-site clarity ("our own site" vs "a competitor's site"), not
|
|
because the matching behavior differs.
|
|
|
|
Data flow:
|
|
website (tenant's own storefront root) + destination/duration/
|
|
travel_style → find_competitor_trip_url() → { url, confidence,
|
|
found }
|
|
"""
|
|
return find_competitor_trip_url(website, destination, duration, travel_style)
|
|
|
|
|
|
def find_all_competitor_urls(competitors, destination, duration, travel_style):
|
|
"""
|
|
Runs trip URL finding for all selected competitors concurrently —
|
|
at most MAX_CONCURRENT_MAP_CALLS Firecrawl /map calls in flight at
|
|
once, respecting server limits.
|
|
|
|
Data flow:
|
|
competitors list → ThreadPoolExecutor(max_workers=5) →
|
|
find_competitor_trip_url() per competitor in parallel →
|
|
collect results as they complete → matches list returned
|
|
(one entry per competitor, in completion order)
|
|
"""
|
|
results = []
|
|
with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_MAP_CALLS) as executor:
|
|
futures = {
|
|
executor.submit(
|
|
find_competitor_trip_url, c['website'], destination, duration, travel_style
|
|
): c
|
|
for c in competitors
|
|
}
|
|
for future in as_completed(futures):
|
|
competitor = futures[future]
|
|
match = future.result()
|
|
results.append({
|
|
'competitor_id': competitor['id'],
|
|
'competitor_name': competitor['name'],
|
|
**match,
|
|
})
|
|
return results
|