239 lines
10 KiB
Python
239 lines
10 KiB
Python
import os
|
|
import logging
|
|
import numpy as np
|
|
from repositories.client_trips_repository import client_trips_repository
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
|
|
EMBEDDING_MODEL = 'openai/text-embedding-3-small'
|
|
SIMILARITY_THRESHOLD = 0.75
|
|
|
|
|
|
def save_client_trip(tenant_id, destination, duration, travel_style, product_name=None,
|
|
url=None, extracted=None, price_usd=None, activities=None):
|
|
"""
|
|
Upserts a client_trips record from a PM's trip-intent description
|
|
(CLAUDE.md §8's TripIntentForm — destination/duration/travel_style,
|
|
optionally a product name). Every /research submission calls this,
|
|
which is the only place client_trips ever gets populated — without
|
|
it, find_comparable_trips() has nothing on the client side to match
|
|
competitor products against.
|
|
|
|
url/extracted/price_usd/activities are optional and only ever passed
|
|
by scrape_own_trip() below, once the tenant's own trip page has
|
|
actually been scraped and extracted — the original typed-values-only
|
|
call site in api.py's /research route passes none of these, so its
|
|
behavior is unchanged.
|
|
|
|
Dedupes on (tenant_id, trip_name) so re-running analysis for the
|
|
same product doesn't create duplicate rows — it just keeps the
|
|
existing one current.
|
|
|
|
Data flow:
|
|
destination/duration/travel_style/product_name (+ optional
|
|
url/extracted/price_usd/activities from a real scrape) →
|
|
trip_name resolved (product_name if given, else a
|
|
destination+style fallback) →
|
|
client_trips_repository.get_by_tenant_and_name() →
|
|
found: update in place → not found: create →
|
|
client_trips record returned
|
|
"""
|
|
trip_name = product_name or f'{destination} {travel_style}'.strip()
|
|
payload = {
|
|
'tenant_id': tenant_id,
|
|
'trip_name': trip_name,
|
|
'destination': destination,
|
|
'duration_days': duration,
|
|
}
|
|
if url is not None:
|
|
payload['url'] = url
|
|
if extracted is not None:
|
|
payload['extracted'] = extracted
|
|
if price_usd is not None:
|
|
payload['price_usd'] = price_usd
|
|
if activities is not None:
|
|
payload['activities'] = activities
|
|
|
|
existing = client_trips_repository.get_by_tenant_and_name(tenant_id, trip_name)
|
|
if existing:
|
|
# A stale embedding would silently never get regenerated —
|
|
# /internal/embed-products only embeds trips missing a vector —
|
|
# so clear it whenever the text embed_trip() hashes actually changed.
|
|
if existing.get('destination') != destination or existing.get('duration_days') != duration:
|
|
payload['embedding'] = None
|
|
return client_trips_repository.update(existing['id'], payload)
|
|
return client_trips_repository.create(payload)
|
|
|
|
|
|
def scrape_own_trip(tenant_id, website_url, context):
|
|
"""
|
|
Scrapes and extracts the tenant's OWN confirmed/matched trip page —
|
|
the same Firecrawl-matched-or-manually-swapped URL pattern already
|
|
used for competitors, pointed at the tenant's own site instead. Gives
|
|
build_prompt()'s own_trip param and client_trips real ground-truth
|
|
data instead of only the PM's typed destination/duration/style.
|
|
|
|
Reuses the same CACHE_STALENESS_HOURS freshness window as competitors
|
|
(core/scraper.py) so the tenant's own site isn't rescraped on every
|
|
single analysis run — only when the cached client_trips row is stale
|
|
or context['force_refresh'] is set, both exactly mirroring
|
|
analysis_service.needs_refresh()'s competitor logic.
|
|
|
|
Never raises — a side-effect failure here (site blocked, proxy
|
|
error, extraction failure) must not fail the whole batch (CLAUDE.md
|
|
§18). Logs and returns None on any failure, which build_prompt()
|
|
treats identically to a tenant who never confirmed an own-trip URL.
|
|
|
|
The returned dict always carries a `cached` bool (True when served
|
|
from client_trips.extracted without a live scrape, False when just
|
|
freshly extracted) — surfaced to the dashboard via jobs.py's
|
|
results['own_trip'] so ResultsTable.vue can show the same "from
|
|
PocketBase vs new scrape" pill on the own-trip column as every
|
|
competitor column. `link` is always overwritten with the real
|
|
website_url actually scraped, rather than trusting the LLM's own
|
|
echoed `"link"` field verbatim — same reasoning as
|
|
analysis_service._save_scrape_result() using page_data['url'], not
|
|
extracted.get('link'), for a competitor's stored product URL.
|
|
|
|
Data flow:
|
|
website_url + context → trip_name resolved exactly as
|
|
save_client_trip() does (product_name, else destination+style) →
|
|
existing client_trips row fresh (and not force_refresh)? → return
|
|
its cached `extracted` blob (cached=True) → otherwise:
|
|
core.scraper.scrape() → build_own_trip_prompt() →
|
|
core.extractor.extract_with_openrouter() → link corrected to the
|
|
real website_url → save_client_trip() with
|
|
url/extracted/price_usd/activities → extracted dict returned
|
|
(cached=False)
|
|
"""
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
trip_name = context.get('product_name') or f"{context.get('destination')} {context.get('travel_style')}".strip()
|
|
existing = client_trips_repository.get_by_tenant_and_name(tenant_id, trip_name)
|
|
force_refresh = context.get('force_refresh', False)
|
|
|
|
if existing and existing.get('extracted') and not force_refresh:
|
|
last_scraped = existing.get('last_scraped')
|
|
if last_scraped:
|
|
try:
|
|
from core.scraper import CACHE_STALENESS_HOURS
|
|
scraped_at = datetime.fromisoformat(last_scraped.replace('Z', '+00:00'))
|
|
if datetime.now(timezone.utc) - scraped_at <= timedelta(hours=CACHE_STALENESS_HOURS):
|
|
logger.info(f'Own trip for tenant {tenant_id} — cache fresh, reusing extracted data')
|
|
return {**existing['extracted'], 'cached': True}
|
|
except (ValueError, AttributeError):
|
|
pass # unparseable timestamp — fall through and rescrape
|
|
|
|
try:
|
|
from core.scraper import scrape
|
|
from core.extractor import extract_with_openrouter
|
|
from industries.adventure_travel.prompt import build_own_trip_prompt
|
|
|
|
page_data = scrape(website_url)
|
|
if not page_data.get('success'):
|
|
logger.warning(f'Own-trip scrape failed for tenant {tenant_id}: {page_data.get("text")}')
|
|
return None
|
|
|
|
prompt = build_own_trip_prompt(
|
|
page_data, context.get('product_name'), context.get('destination'),
|
|
context.get('duration'), context.get('travel_style'),
|
|
context.get('company_name'), is_ngs=context.get('tab_type') == 'NGS',
|
|
)
|
|
extracted = extract_with_openrouter(prompt, content=page_data['text'])
|
|
extracted['link'] = website_url
|
|
|
|
saved = save_client_trip(
|
|
tenant_id, context.get('destination'), context.get('duration'),
|
|
context.get('travel_style'), product_name=context.get('product_name'),
|
|
url=website_url, extracted=extracted,
|
|
price_usd=extracted.get('majorityPrice'), activities=extracted.get('activities'),
|
|
)
|
|
client_trips_repository.update(saved['id'], {'last_scraped': datetime.now(timezone.utc).isoformat()})
|
|
return {**extracted, 'cached': False}
|
|
except Exception as e:
|
|
logger.error(f'scrape_own_trip failed for tenant {tenant_id}: {str(e)}')
|
|
return None
|
|
|
|
|
|
def embed_trip(trip):
|
|
"""
|
|
Generates a semantic embedding for a trip using OpenRouter's
|
|
text-embedding-3-small, combining trip name, destination, duration,
|
|
and activities into a single text representation first.
|
|
|
|
Data flow:
|
|
trip dict → formatted text string → OpenRouter embeddings API →
|
|
vector list returned for storage in products.embedding /
|
|
client_trips.embedding
|
|
"""
|
|
# Imported lazily so this module has no hard dependency on the
|
|
# `openai` package at import time for callers that only need
|
|
# find_comparable_trips() (pure math, no API calls).
|
|
from openai import OpenAI
|
|
|
|
text = (
|
|
f"{trip.get('trip_name', '')}\n"
|
|
f"Destination: {trip.get('destination', '')}\n"
|
|
f"Duration: {trip.get('duration_days', '')} days\n"
|
|
f"Activities: {trip.get('activities', '')}\n"
|
|
f"Start: {trip.get('start_location', '')}"
|
|
)
|
|
|
|
client = OpenAI(api_key=OPENROUTER_API_KEY, base_url='https://openrouter.ai/api/v1')
|
|
response = client.embeddings.create(model=EMBEDDING_MODEL, input=text)
|
|
return response.data[0].embedding
|
|
|
|
|
|
def _cosine_similarity(a, b):
|
|
"""Standard cosine similarity between two equal-length embedding vectors."""
|
|
a, b = np.array(a), np.array(b)
|
|
denom = (np.linalg.norm(a) * np.linalg.norm(b))
|
|
if denom == 0:
|
|
return 0.0
|
|
return float(np.dot(a, b) / denom)
|
|
|
|
|
|
def find_comparable_trips(client_trips, competitor_trips, threshold=SIMILARITY_THRESHOLD):
|
|
"""
|
|
Calculates cosine similarity between every client trip and every
|
|
competitor trip, returning matches above threshold sorted by
|
|
similarity score descending.
|
|
|
|
Data flow:
|
|
client_trips + competitor_trips (each with .embedding) →
|
|
pairwise cosine similarity → filter by threshold →
|
|
sort by score descending → list of match dicts returned
|
|
"""
|
|
matches = []
|
|
for client_trip in client_trips:
|
|
client_embedding = client_trip.get('embedding')
|
|
if not client_embedding:
|
|
continue
|
|
for competitor_trip in competitor_trips:
|
|
competitor_embedding = competitor_trip.get('embedding')
|
|
if not competitor_embedding:
|
|
continue
|
|
score = _cosine_similarity(client_embedding, competitor_embedding)
|
|
if score >= threshold:
|
|
matches.append({
|
|
'client_product': client_trip.get('trip_name'),
|
|
'competitor_product': competitor_trip.get('id'),
|
|
'similarity_score': round(score, 4),
|
|
'price_difference': _safe_diff(
|
|
competitor_trip.get('majority_price_usd'), client_trip.get('price_usd')
|
|
),
|
|
'duration_difference': _safe_diff(
|
|
competitor_trip.get('duration_days'), client_trip.get('duration_days')
|
|
),
|
|
})
|
|
matches.sort(key=lambda m: m['similarity_score'], reverse=True)
|
|
return matches
|
|
|
|
|
|
def _safe_diff(a, b):
|
|
"""Returns a - b if both are present, else None — keeps the diff fields nullable per schema."""
|
|
if a is None or b is None:
|
|
return None
|
|
return a - b
|