115 lines
4.1 KiB
Python
115 lines
4.1 KiB
Python
import os
|
|
import logging
|
|
import requests
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FIRECRAWL_URL = os.getenv('FIRECRAWL_URL', 'http://firecrawl:3002')
|
|
|
|
CATEGORY_KEYWORDS = ['classic', 'trek', 'trail', 'safari', 'adventure', 'explorer', 'journey']
|
|
CONFIDENCE_FOUND_THRESHOLD = 0.3
|
|
MAX_CONCURRENT_MAP_CALLS = 5
|
|
|
|
|
|
def score_and_rank_urls(urls, destination, duration):
|
|
"""
|
|
Scores candidate URLs by keyword match against destination and
|
|
duration, returning the highest-scoring URL with a confidence score.
|
|
|
|
Data flow:
|
|
urls + destination + duration →
|
|
per-url score: +1 per destination keyword in URL, +2 if duration
|
|
appears in URL, +1 per travel-category keyword in URL →
|
|
sort by score → confidence = top_score / max_possible_score →
|
|
{ url, confidence, found } — found=False (prompt manual entry)
|
|
when confidence is below CONFIDENCE_FOUND_THRESHOLD
|
|
"""
|
|
dest_keywords = destination.lower().split()
|
|
scores = []
|
|
|
|
for url in urls:
|
|
url_lower = url.lower()
|
|
score = 0
|
|
score += sum(1 for kw in dest_keywords if kw in url_lower)
|
|
if str(duration) in url_lower:
|
|
score += 2
|
|
score += sum(1 for kw in CATEGORY_KEYWORDS if kw in url_lower)
|
|
scores.append((score, url))
|
|
|
|
scores.sort(key=lambda pair: pair[0], reverse=True)
|
|
if not scores or scores[0][0] == 0:
|
|
return {'url': None, 'confidence': 0, 'found': False}
|
|
|
|
top_score, top_url = scores[0]
|
|
max_possible = len(dest_keywords) + 2 + 1
|
|
confidence = min(top_score / max_possible, 1.0) if max_possible else 0
|
|
|
|
return {
|
|
'url': top_url,
|
|
'confidence': round(confidence, 2),
|
|
'found': confidence >= CONFIDENCE_FOUND_THRESHOLD,
|
|
}
|
|
|
|
|
|
def find_competitor_trip_url(root_url, destination, duration, travel_style):
|
|
"""
|
|
Uses Firecrawl /map with search terms to find the most likely trip
|
|
page on a competitor site matching the PM's research intent.
|
|
|
|
Data flow:
|
|
root_url + destination/duration/travel_style →
|
|
POST {FIRECRAWL_URL}/v1/map with a search string →
|
|
candidate URL list → score_and_rank_urls() →
|
|
{ url, confidence, found } returned (found=False, url=None on any
|
|
Firecrawl failure — never raises, since one competitor's failure
|
|
must not block the others per CLAUDE.md §18 resilience rules)
|
|
"""
|
|
try:
|
|
response = requests.post(
|
|
f'{FIRECRAWL_URL}/v1/map',
|
|
json={'url': root_url, 'search': f'{destination} {duration} days {travel_style} tour'},
|
|
timeout=15
|
|
)
|
|
response.raise_for_status()
|
|
urls = response.json().get('links', [])
|
|
except requests.RequestException as e:
|
|
logger.error(f'Firecrawl /map failed for {root_url}: {str(e)}')
|
|
return {'url': None, 'confidence': 0, 'found': False}
|
|
|
|
if not urls:
|
|
return {'url': None, 'confidence': 0, 'found': False}
|
|
|
|
return score_and_rank_urls(urls, destination, duration)
|
|
|
|
|
|
def find_all_competitor_urls(competitors, destination, duration, travel_style):
|
|
"""
|
|
Runs trip URL finding for all selected competitors concurrently —
|
|
at most MAX_CONCURRENT_MAP_CALLS Firecrawl /map calls in flight at
|
|
once, respecting server limits.
|
|
|
|
Data flow:
|
|
competitors list → ThreadPoolExecutor(max_workers=5) →
|
|
find_competitor_trip_url() per competitor in parallel →
|
|
collect results as they complete → matches list returned
|
|
(one entry per competitor, in completion order)
|
|
"""
|
|
results = []
|
|
with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_MAP_CALLS) as executor:
|
|
futures = {
|
|
executor.submit(
|
|
find_competitor_trip_url, c['website'], destination, duration, travel_style
|
|
): c
|
|
for c in competitors
|
|
}
|
|
for future in as_completed(futures):
|
|
competitor = futures[future]
|
|
match = future.result()
|
|
results.append({
|
|
'competitor_id': competitor['id'],
|
|
'competitor_name': competitor['name'],
|
|
**match,
|
|
})
|
|
return results
|